00010000:00000010:0.1:1775770559.297166:0:166678:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2400. 00010000:00000010:0.1:1775770559.558265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200031000. 00010000:00000010:0.1:1775770559.587197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220673800. 00010000:00000010:0.1:1775770559.674690:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcec00. 00010000:00000010:0.1:1775770560.015180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44200. 00010000:00000010:0.1:1775770560.326810:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a8600. 00010000:00000010:0.1:1775770560.326816:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a9200. 00010000:00000010:0.1:1775770560.421718:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5d000. 00010000:00000010:0.1:1775770560.450168:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cd600. 00010000:00000010:0.1:1775770560.639295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcfa00. 00010000:00000010:0.1:1775770560.742227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6c00. 00010000:00000010:0.1:1775770561.363756:0:181430:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c6bbc00. 00010000:00000010:0.1:1775770561.371596:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6209e73800. 00010000:00000010:0.1:1775770561.371614:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205045e00. 00010000:00000010:0.1:1775770561.544111:0:166678:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6400. 00010000:00000010:0.1:1775770561.559113:0:166678:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2e00. 00010000:00000010:0.1:1775770561.559119:0:166678:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45200. 00010000:00000010:0.1:1775770561.865129:0:181435:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45c00. 00010000:00000010:0.1:1775770561.865152:0:181435:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207946a00. 00010000:00000010:0.1:1775770561.880488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205044e00. 00010000:00000010:0.1:1775770561.880495:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcfc00. 00010000:00000010:0.1:1775770562.051714:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44e00. 00010000:00000010:0.1:1775770562.401372:0:181432:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dc800. 00010000:00000010:0.1:1775770562.450786:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cc400. 00010000:00000010:0.1:1775770562.770181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210792c00. 00010000:00000010:0.1:1775770562.825293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e186400. 00010000:00000010:0.1:1775770562.892319:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205044200. 00010000:00000010:0.1:1775770563.023067:0:181430:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210792800. 00010000:00000010:0.1:1775770563.206316:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167f800. 00010000:00000010:0.1:1775770563.619160:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2800. 00010000:00000010:0.1:1775770563.619166:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205044200. 00010000:00000010:0.1:1775770563.662250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e186a00. 00010000:00000010:0.1:1775770563.751191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e187e00. 00010000:00000010:0.1:1775770563.971485:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205045e00. 00010000:00000010:0.1:1775770563.971504:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcfc00. 00010000:00000010:0.1:1775770563.990165:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3400. 00010000:00000010:0.1:1775770564.131634:0:181434:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205045400. 00010000:00000010:0.1:1775770564.145404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e322c00. 00010000:00000010:0.1:1775770564.316177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49ee00. 00010000:00000010:0.1:1775770564.378973:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bce800. 00010000:00000010:0.1:1775770564.383186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57b200. 00010000:00000010:0.1:1775770564.481819:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7400. 00010000:00000010:0.1:1775770564.662920:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcf200. 00010000:00000010:0.1:1775770564.700647:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49fc00. 00010000:00000010:0.1:1775770564.796452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3800. 00010000:00000010:0.1:1775770564.899286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3a00. 00010000:00000010:0.1:1775770564.901674:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207947800. 00010000:00000010:0.1:1775770565.116217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6600. 00010000:00000010:0.1:1775770565.116229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5dc00. 00010000:00000010:0.1:1775770565.156173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5d800. 00010000:00000010:0.1:1775770565.466036:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e186e00. 00010000:00000010:0.1:1775770565.499237:0:181425:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45800. 00010000:00000010:0.1:1775770565.662178:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7800. 00010000:00000010:0.1:1775770565.662185:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201528800. 00010000:00000010:0.1:1775770565.826158:0:181419:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6200. 00010000:00000010:0.1:1775770565.957903:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2400. 00010000:00000010:0.1:1775770566.050415:0:181433:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5d400. 00010000:00000010:0.1:1775770566.314693:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207947e00. 00010000:00000010:0.1:1775770566.375448:0:168160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcec00. 00010000:00000010:0.1:1775770566.375454:0:168160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167fa00. 00010000:00000010:0.1:1775770566.375461:0:168160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49e200. 00010000:00000010:0.1:1775770566.393480:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2200. 00010000:00000010:0.1:1775770566.447978:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5c200. 00010000:00000010:0.1:1775770566.713115:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207946000. 00010000:00000010:0.1:1775770566.713162:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5ca00. 00010000:00000010:0.1:1775770566.794176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49e000. 00010000:00000010:0.1:1775770566.906722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207947400. 00010000:00000010:0.1:1775770566.969485:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5d200. 00010000:00000010:0.1:1775770567.029868:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49fe00. 00010000:00000010:0.1:1775770567.088033:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5d400. 00010000:00000010:0.1:1775770567.088044:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cd400. 00010000:00000010:0.1:1775770567.126156:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200030800. 00010000:00000010:0.1:1775770567.217847:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5d400. 00010000:00000010:0.1:1775770567.299794:0:181430:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49e200. 00010000:00000010:0.1:1775770567.531049:0:181430:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5c000. 00010000:00000010:0.1:1775770567.553850:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220673a00. 00010000:00000010:0.1:1775770567.625352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207946e00. 00010000:00000010:0.1:1775770567.915157:0:171818:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bce800. 00010000:00000010:0.1:1775770567.927978:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220673c00. 00010000:00000010:0.1:1775770568.104941:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207947c00. 00010000:00000010:0.1:1775770568.163177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220673c00. 00010000:00000010:0.1:1775770568.331114:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57aa00. 00010000:00000010:0.1:1775770568.467236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207947e00. 00010000:00000010:0.1:1775770568.467240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49f400. 00010000:00000010:0.1:1775770568.484184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49ea00. 00010000:00000010:0.1:1775770568.484201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57a800. 00010000:00000010:0.1:1775770568.510250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57ba00. 00010000:00000010:0.1:1775770568.661795:0:166678:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57b000. 00010000:00000010:0.1:1775770568.889493:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c6ba200. 00010000:00000010:0.1:1775770568.956216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57a200. 00010000:00000010:0.1:1775770569.040546:0:181430:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5c200. 00010000:00000010:0.1:1775770569.051040:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220672a00. 00010000:00000010:0.1:1775770569.051056:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207946a00. 00010000:00000010:0.1:1775770569.194281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207946a00. 00010000:00000010:0.1:1775770569.402212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c6ba200. 00010000:00000010:0.1:1775770569.550494:0:181420:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210793200. 00010000:00000010:0.1:1775770569.706713:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201529600. 00010000:00000010:0.1:1775770569.759185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c6bae00. 00010000:00000010:0.1:1775770569.957400:0:1084:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6400. 00010000:00000010:0.1:1775770570.257290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57b000. 00010000:00000010:0.1:1775770570.341353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207947e00. 00010000:00000010:0.1:1775770570.341368:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2000. 00010000:00000010:0.1:1775770570.520279:0:166678:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5d400. 00010000:00000010:0.1:1775770570.541165:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cc000. 00010000:00000010:0.1:1775770570.779311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201528a00. 00010000:00000010:0.1:1775770570.852225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2600. 00010000:00000010:0.1:1775770570.906503:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3600. 00010000:00000010:0.1:1775770571.213723:0:181432:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5ca00. 00010000:00000010:0.1:1775770571.543495:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6600. 00010000:00000010:0.1:1775770571.734181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cdc00. 00010000:00000010:0.1:1775770571.908188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205045c00. 00010000:00000010:0.1:1775770572.119636:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210792c00. 00010000:00000010:0.1:1775770572.124179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205045000. 00010000:00000010:0.1:1775770572.193162:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57b400. 00010000:00000010:0.1:1775770572.218162:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210793e00. 00010000:00000010:0.1:1775770572.218167:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c6bac00. 00010000:00000010:0.1:1775770572.394916:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44a00. 00010000:00000010:0.1:1775770572.409174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2000. 00010000:00000010:0.1:1775770572.417538:0:181432:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3000. 00010000:00000010:0.1:1775770572.417549:0:181432:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5ce00. 00010000:00000010:0.1:1775770572.501302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6209e72e00. 00010000:00000010:0.1:1775770572.670457:0:181438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207946a00. 00010000:00000010:0.1:1775770572.759168:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5d600. 00010000:00000010:0.1:1775770572.759192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e186200. 00010000:00000010:0.1:1775770572.759195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323400. 00010000:00000010:0.1:1775770572.759198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5cc00. 00010000:00000010:0.1:1775770572.809551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5ce00. 00010000:00000010:0.1:1775770572.981520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e186600. 00010000:00000010:0.1:1775770572.981534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200031000. 00010000:00000010:0.1:1775770573.162371:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcea00. 00010000:00000010:0.1:1775770573.391859:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167e000. 00010000:00000010:0.1:1775770573.457319:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200030800. 00010000:00000010:0.1:1775770573.457326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207946600. 00010000:00000010:0.1:1775770573.611149:0:181426:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49ec00. 00010000:00000010:0.1:1775770573.649309:0:181434:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210792000. 00010000:00000010:0.1:1775770573.875747:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323a00. 00010000:00000010:0.1:1775770574.172238:0:171818:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e186000. 00010000:00000010:0.1:1775770574.263197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e186800. 00010000:00000010:0.1:1775770574.406256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e186000. 00010000:00000010:0.1:1775770574.521866:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207946200. 00010000:00000010:0.1:1775770574.860339:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e186c00. 00010000:00000010:0.1:1775770575.650253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220672c00. 00010000:00000010:0.1:1775770575.756699:0:181432:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6209e73800. 00010000:00000010:0.1:1775770575.862381:0:168160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57a400. 00010000:00000010:0.1:1775770575.862387:0:168160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6209e72600. 00010000:00000010:0.1:1775770575.862390:0:168160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c6bb600. 00010000:00000010:0.1:1775770575.862393:0:168160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205045200. 00010000:00000010:0.1:1775770575.862397:0:168160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2400. 00010000:00000010:0.1:1775770575.862401:0:168160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcec00. 00010000:00000010:0.1:1775770575.862404:0:168160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e322000. 00010000:00000010:0.1:1775770575.862406:0:168160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323600. 00010000:00000010:0.1:1775770575.862409:0:168160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207946c00. 00010000:00000010:0.1:1775770575.862412:0:168160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3600. 00010000:00000010:0.1:1775770575.862419:0:168160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bce000. 00010000:00000010:0.1:1775770575.868216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57bc00. 00010000:00000010:0.1:1775770576.015179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57bc00. 00010000:00000010:0.1:1775770576.080740:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cde00. 00010000:00000010:0.1:1775770576.303250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e322400. 00010000:00000010:0.1:1775770576.329242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207946e00. 00010000:00000010:0.1:1775770576.359855:0:180656:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57b800. 00010000:00000010:0.1:1775770576.559188:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207947800. 00010000:00000010:0.1:1775770576.559195:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6e00. 00010000:00000010:0.1:1775770576.598192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcfe00. 00010000:00000010:0.1:1775770576.598202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcee00. 00010000:00000010:0.1:1775770576.864301:0:181433:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcea00. 00010000:00000010:0.1:1775770577.100222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167f400. 00010000:00000010:0.1:1775770577.221225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c6ba000. 00010000:00000010:0.1:1775770577.223409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201529800. 00010000:00000010:0.1:1775770577.468215:0:181426:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49e200. 00010000:00000010:0.1:1775770577.480241:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167e000. 00010000:00000010:0.1:1775770577.480254:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcf800. 00010000:00000010:0.1:1775770577.704177:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6e00. 00010000:00000010:0.1:1775770577.732976:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323000. 00010000:00000010:0.1:1775770577.759470:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a8c00. 00010000:00000010:0.1:1775770577.927279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6209e72400. 00010000:00000010:0.1:1775770578.002239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cd600. 00010000:00000010:0.1:1775770578.002251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bce200. 00010000:00000010:0.1:1775770578.387324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57aa00. 00010000:00000010:0.1:1775770578.416165:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44c00. 00010000:00000010:0.1:1775770578.518047:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c6bbe00. 00010000:00000010:0.1:1775770578.518059:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49e600. 00010000:00000010:0.1:1775770578.524476:0:181438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a8200. 00010000:00000010:0.1:1775770578.524481:0:181438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205044200. 00010000:00000010:0.1:1775770578.524484:0:181438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7c00. 00010000:00000010:0.1:1775770578.561161:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bce400. 00010000:00000010:0.1:1775770578.561167:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e187a00. 00010000:00000010:0.1:1775770578.561170:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a9000. 00010000:00000010:0.1:1775770578.705200:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49f200. 00010000:00000010:0.1:1775770578.705206:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220672c00. 00010000:00000010:0.1:1775770578.737539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5d200. 00010000:00000010:0.1:1775770578.888199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e322800. 00010000:00000010:0.1:1775770579.021121:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49f600. 00010000:00000010:0.1:1775770579.039420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44200. 00010000:00000010:0.1:1775770579.146195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323e00. 00010000:00000010:0.1:1775770579.161553:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45800. 00010000:00000010:0.1:1775770579.229201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57b400. 00010000:00000010:0.1:1775770579.343210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcfc00. 00010000:00000010:0.1:1775770579.365207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57a400. 00010000:00000010:0.1:1775770579.559495:0:181433:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57b800. 00010000:00000010:0.1:1775770579.578179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6a00. 00010000:00000010:0.1:1775770579.674994:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57b000. 00010000:00000010:0.1:1775770579.887200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a9800. 00010000:00000010:0.1:1775770579.931322:0:181432:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45400. 00010000:00000010:0.1:1775770580.165807:0:180656:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323200. 00010000:00000010:0.1:1775770580.200175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a8200. 00010000:00000010:0.1:1775770580.326844:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323c00. 00010000:00000010:0.1:1775770580.397939:0:181432:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167ec00. 00010000:00000010:0.1:1775770580.513202:0:181433:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323a00. 00010000:00000010:0.1:1775770580.513214:0:181433:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49f200. 00010000:00000010:0.1:1775770580.627169:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a8000. 00010000:00000010:0.1:1775770580.693984:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49f600. 00010000:00000010:0.1:1775770580.873235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6209e73600. 00010000:00000010:0.1:1775770580.972179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323400. 00010000:00000010:0.1:1775770581.037200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49e200. 00010000:00000010:0.1:1775770581.131002:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6209e73400. 00010000:00000010:0.1:1775770581.246628:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6600. 00010000:00000010:0.1:1775770581.251653:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49ec00. 00010000:00000010:0.1:1775770581.328639:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcea00. 00010000:00000010:0.1:1775770581.335251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a8a00. 00010000:00000010:0.1:1775770581.428302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323200. 00010000:00000010:0.1:1775770581.486228:0:181425:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323000. 00010000:00000010:0.1:1775770581.556771:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49ec00. 00010000:00000010:0.1:1775770581.632185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167f800. 00010000:00000010:0.1:1775770581.707414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7000. 00010000:00000010:0.1:1775770581.707425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7e00. 00010000:00000010:0.1:1775770581.832178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49e600. 00010000:00000010:0.1:1775770581.837801:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49fa00. 00010000:00000010:0.1:1775770581.837809:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220672000. 00010000:00000010:0.1:1775770581.837812:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57aa00. 00010000:00000010:0.1:1775770581.976497:0:181432:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6209e73200. 00010000:00000010:0.1:1775770582.044675:0:155797:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e322400. 00010000:00000010:0.1:1775770582.094050:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167e200. 00010000:00000010:0.1:1775770582.322252:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167f000. 00010000:00000010:0.1:1775770582.322261:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323e00. 00010000:00000010:0.1:1775770582.361233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57a400. 00010000:00000010:0.1:1775770582.365187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cd200. 00010000:00000010:0.1:1775770582.526503:0:168160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e322e00. 00010000:00000010:0.1:1775770582.538232:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6e00. 00010000:00000010:0.1:1775770582.543469:0:181428:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207947400. 00010000:00000010:0.1:1775770582.716159:0:181433:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207947e00. 00010000:00000010:0.1:1775770582.835188:0:171818:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e322e00. 00010000:00000010:0.1:1775770582.880580:0:171818:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45c00. 00010000:00000010:0.1:1775770582.880587:0:171818:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167ee00. 00010000:00000010:0.1:1775770582.904161:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6c00. 00010000:00000010:0.1:1775770583.069153:0:180656:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167fc00. 00010000:00000010:0.1:1775770583.113667:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167f200. 00010000:00000010:0.1:1775770583.117449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7c00. 00010000:00000010:0.1:1775770583.117461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323400. 00010000:00000010:0.1:1775770583.187010:0:181430:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7e00. 00010000:00000010:0.1:1775770583.248164:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7c00. 00010000:00000010:0.1:1775770583.326266:0:181428:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200031000. 00010000:00000010:0.1:1775770583.333275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207946600. 00010000:00000010:0.1:1775770583.358357:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200030000. 00010000:00000010:0.1:1775770583.401006:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3200. 00010000:00000010:0.1:1775770583.491287:0:181431:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e322400. 00010000:00000010:0.1:1775770583.541202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7200. 00010000:00000010:0.1:1775770583.678803:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cda00. 00010000:00000010:0.1:1775770583.795649:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167ee00. 00010000:00000010:0.1:1775770584.671563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3800. 00010000:00000010:0.1:1775770584.766176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49f800. 00010000:00000010:0.1:1775770584.802698:0:181431:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205045800. 00010000:00000010:0.1:1775770584.857207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a8a00. 00010000:00000010:0.1:1775770584.881755:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5d000. 00010000:00000010:0.1:1775770584.954432:0:181433:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e186600. 00010000:00000010:0.1:1775770585.079720:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cd200. 00010000:00000010:0.1:1775770585.216451:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6e00. 00010000:00000010:0.1:1775770585.265295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5c600. 00010000:00000010:0.1:1775770585.378285:0:171818:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167fc00. 00010000:00000010:0.1:1775770585.778522:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e322c00. 00010000:00000010:0.1:1775770585.778529:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5c000. 00010000:00000010:0.1:1775770585.967425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5c800. 00010000:00000010:0.1:1775770586.226975:0:171818:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200030600. 00010000:00000010:0.1:1775770586.658165:0:181430:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e187600. 00010000:00000010:0.1:1775770586.658169:0:181430:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323600. 00010000:00000010:0.1:1775770586.775268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200030800. 00010000:00000010:0.1:1775770586.806155:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220673600. 00010000:00000010:0.1:1775770586.841307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e187800. 00010000:00000010:0.1:1775770586.841314:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167f800. 00010000:00000010:0.1:1775770586.980210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200030000. 00010000:00000010:0.1:1775770586.980230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e322e00. 00010000:00000010:0.1:1775770587.198480:0:181431:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3800. 00010000:00000010:0.1:1775770587.234323:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e322a00. 00010000:00000010:0.1:1775770587.464915:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323600. 00010000:00000010:0.1:1775770587.464921:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3a00. 00010000:00000010:0.1:1775770587.563560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5cc00. 00010000:00000010:0.1:1775770587.605239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2400. 00010000:00000010:0.1:1775770587.736093:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200031e00. 00010000:00000010:0.1:1775770587.754376:0:181430:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57ae00. 00010000:00000010:0.1:1775770587.929168:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5d600. 00010000:00000010:0.1:1775770588.139180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5d800. 00010000:00000010:0.1:1775770588.213173:0:181420:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cde00. 00010000:00000010:0.1:1775770588.249207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5c600. 00010000:00000010:0.1:1775770588.532155:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207946a00. 00010000:00000010:0.1:1775770588.588178:0:181433:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e322a00. 00010000:00000010:0.1:1775770589.004444:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2e00. 00010000:00000010:0.1:1775770589.098175:0:181428:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207946400. 00010000:00000010:0.1:1775770589.265092:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6c00. 00010000:00000010:0.1:1775770589.265100:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5de00. 00010000:00000010:0.1:1775770589.283127:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5c400. 00010000:00000010:0.1:1775770589.344300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5c600. 00010000:00000010:0.1:1775770589.344308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e187c00. 00010000:00000010:0.1:1775770589.378285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323c00. 00010000:00000010:0.1:1775770589.408320:0:155798:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5d200. 00010000:00000010:0.1:1775770589.499500:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207947800. 00010000:00000010:0.1:1775770589.512276:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5ce00. 00010000:00000010:0.1:1775770590.071786:0:181428:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167e000. 00010000:00000010:0.1:1775770590.352202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cc800. 00010000:00000010:0.1:1775770590.517745:0:181432:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6c00. 00010000:00000010:0.1:1775770590.946548:0:181420:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49e600. 00010000:00000010:0.1:1775770591.179222:0:180656:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5dc00. 00010000:00000010:0.1:1775770591.179227:0:180656:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5de00. 00010000:00000010:0.1:1775770591.179237:0:180656:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bce800. 00010000:00000010:0.1:1775770591.179244:0:180656:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6200. 00010000:00000010:0.1:1775770591.179247:0:180656:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57a600. 00010000:00000010:0.1:1775770591.519327:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44a00. 00010000:00000010:0.1:1775770591.740278:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcf000. 00010000:00000010:0.1:1775770591.757832:0:181431:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5d000. 00010000:00000010:0.1:1775770591.962302:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201528600. 00010000:00000010:0.1:1775770592.120518:0:166678:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57b400. 00010000:00000010:0.1:1775770592.221971:0:181431:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44e00. 00010000:00000010:0.1:1775770592.351794:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205044400. 00010000:00000010:0.1:1775770592.481159:0:181430:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e322000. 00010000:00000010:0.1:1775770592.517067:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bce600. 00010000:00000010:0.1:1775770592.576214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167e400. 00010000:00000010:0.1:1775770592.576226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201528600. 00010000:00000010:0.1:1775770592.782905:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45a00. 00010000:00000010:0.1:1775770592.871841:0:181426:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dde00. 00010000:00000010:0.1:1775770592.958965:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49f000. 00010000:00000010:0.1:1775770593.195549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201529800. 00010000:00000010:0.1:1775770593.232448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167ec00. 00010000:00000010:0.1:1775770593.380306:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e611a00. 00010000:00000010:0.1:1775770593.456291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45e00. 00010000:00000010:0.1:1775770593.875359:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec800. 00010000:00000010:0.1:1775770593.926207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcf000. 00010000:00000010:0.1:1775770593.926216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201528c00. 00010000:00000010:0.1:1775770594.675942:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6800. 00010000:00000010:0.1:1775770595.726183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e57a600. 00010000:00000010:0.1:1775770595.766223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e611200. 00010000:00000010:0.1:1775770595.889174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e610000. 00010000:00000010:0.1:1775770595.889180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5c600. 00010000:00000010:0.1:1775770596.096862:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e610000. 00010000:00000010:0.1:1775770596.124187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ece00. 00010000:00000010:0.1:1775770596.228462:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5c800. 00010000:00000010:0.1:1775770596.466923:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cca00. 00010000:00000010:0.1:1775770596.561387:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcfe00. 00010000:00000010:0.1:1775770596.929251:0:181428:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e611e00. 00010000:00000010:0.1:1775770597.185245:0:168160:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dc800. 00010000:00000010:0.1:1775770597.194244:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec600. 00010000:00000010:0.1:1775770597.194250:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266ccc00. 00010000:00000010:0.1:1775770597.194253:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44200. 00010000:00000010:0.1:1775770597.194258:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e49e400. 00010000:00000010:0.1:1775770597.245218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e611000. 00010000:00000010:0.1:1775770597.752220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207947c00. 00010000:00000010:0.1:1775770597.752229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e611400. 00010000:00000010:0.1:1775770597.781976:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5d200. 00010000:00000010:0.1:1775770597.817377:0:181420:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b5ca00. 00010000:00000010:0.1:1775770597.967701:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a9600. 00010000:00000010:0.1:1775770598.236377:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cd000. 00010000:00000010:0.1:1775770598.783408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec000. 00010000:00000010:0.1:1775770598.900963:0:181431:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e611000. 00010000:00000010:0.1:1775770598.963575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cc200. 00010000:00000010:0.1:1775770599.098283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207947a00. 00010000:00000010:0.1:1775770599.327381:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cdc00. 00010000:00000010:0.1:1775770599.327393:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cda00. 00010000:00000010:0.1:1775770599.513944:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cd000. 00010000:00000010:0.1:1775770599.828651:0:181432:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207946c00. 00010000:00000010:0.1:1775770599.849865:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3200. 00010000:00000010:0.1:1775770599.866508:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e610a00. 00010000:00000010:0.1:1775770599.924887:0:155797:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345edc00. 00010000:00000010:0.1:1775770600.114011:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2200. 00010000:00000010:0.1:1775770600.445204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2200. 00010000:00000010:0.1:1775770601.196294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45800. 00010000:00000010:0.1:1775770601.196301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e610200. 00010000:00000010:0.1:1775770601.287356:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cc600. 00010000:00000010:0.1:1775770601.714213:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ede00. 00010000:00000010:0.1:1775770601.775199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3800. 00010000:00000010:0.1:1775770601.861187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323800. 00010000:00000010:0.1:1775770601.978235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e611200. 00010000:00000010:0.1:1775770603.331185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e610c00. 00010000:00000010:0.1:1775770604.122858:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7e00. 00010000:00000010:0.1:1775770606.665417:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bce800. 00010000:00000010:0.1:1775770607.419205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cd600. 00000080:00000010:0.1:1775770656.499768:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39180. 00010000:00000010:0.1:1775770657.083186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3e00. 00010000:00000010:0.1:1775770657.130226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3400. 00010000:00000010:0.1:1775770657.135449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323400. 00010000:00000010:0.1:1775770657.135463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3800. 00010000:00000010:0.1:1775770657.181181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cc200. 00010000:00000010:0.1:1775770657.202201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323000. 00010000:00000010:0.1:1775770657.202215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3600. 00010000:00000010:0.1:1775770657.206198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e611a00. 00010000:00000010:0.1:1775770657.274532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2000. 00010000:00000010:0.1:1775770657.359313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2000. 00000080:00000010:0.1:1775770657.400258:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c6cc440. 00010000:00000010:0.1:1775770657.436640:0:155798:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cc400. 00000080:00000010:0.1:1775770657.436648:0:155798:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6200025060. 00010000:00000010:0.1:1775770657.494545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2a00. 00010000:00000010:0.1:1775770657.648961:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e610000. 00010000:00000010:0.1:1775770657.858478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3000. 00010000:00000010:0.1:1775770658.056572:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3600. 00010000:00000010:0.1:1775770658.086157:0:181475:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7600. 00010000:00000010:0.1:1775770658.240259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3400. 00010000:00000010:0.1:1775770658.474182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205044800. 00010000:00000010:0.1:1775770658.855081:0:181466:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323600. 00010000:00000010:0.1:1775770658.923188:0:155797:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3400. 00010000:00000010:0.1:1775770659.222125:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cc400. 00010000:00000010:0.1:1775770659.521730:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e611600. 00010000:00000010:0.1:1775770659.663823:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e611800. 00010000:00000010:0.1:1775770659.895224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3c00. 00010000:00000010:0.1:1775770660.301911:0:155798:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e322a00. 00010000:00000010:0.1:1775770660.626801:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e186e00. 00010000:00000010:0.1:1775770660.685987:0:171818:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e187000. 00010000:00000010:0.1:1775770660.728959:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2800. 00010000:00000010:0.1:1775770660.871438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2200. 00010000:00000010:0.1:1775770661.071755:0:181472:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e322c00. 00010000:00000010:0.1:1775770661.071766:0:181472:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e611a00. 00010000:00000010:0.1:1775770661.182820:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205044e00. 00010000:00000010:0.1:1775770661.479786:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200030400. 00010000:00000010:0.1:1775770661.572544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e187a00. 00010000:00000010:0.1:1775770661.839859:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205045600. 00010000:00000010:0.1:1775770661.924213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2e00. 00010000:00000010:0.1:1775770662.029652:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e322000. 00010000:00000010:0.1:1775770662.103194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200030a00. 00010000:00000010:0.1:1775770662.223165:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205045e00. 00010000:00000010:0.1:1775770662.451876:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205044200. 00010000:00000010:0.1:1775770662.471879:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201529400. 00010000:00000010:0.1:1775770662.763553:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2800. 00010000:00000010:0.1:1775770662.763565:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323800. 00010000:00000010:0.1:1775770662.763574:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e610a00. 00010000:00000010:0.1:1775770662.766887:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205044c00. 00010000:00000010:0.1:1775770662.766909:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e610000. 00010000:00000010:0.1:1775770662.766914:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cc800. 00010000:00000010:0.1:1775770662.767032:0:166678:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cd800. 00010000:00000010:0.1:1775770662.830806:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266ccc00. 00010000:00000010:0.1:1775770662.952224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cdc00. 00010000:00000010:0.1:1775770663.017680:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323800. 00010000:00000010:0.1:1775770663.178094:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e611a00. 00010000:00000010:0.1:1775770663.178102:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205044400. 00010000:00000010:0.1:1775770663.178106:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e610c00. 00010000:00000010:0.1:1775770663.198310:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2600. 00010000:00000010:0.1:1775770663.198319:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201529800. 00010000:00000010:0.1:1775770663.602237:0:181466:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e610e00. 00010000:00000010:0.1:1775770663.609545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e187200. 00010000:00000010:0.1:1775770664.066175:0:181474:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266ccc00. 00010000:00000010:0.1:1775770664.209626:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e610c00. 00010000:00000010:0.1:1775770664.393170:0:171818:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ed200. 00010000:00000010:0.1:1775770664.393248:0:171818:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cd000. 00010000:00000010:0.1:1775770664.409692:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e611e00. 00010000:00000010:0.1:1775770664.537210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cd600. 00010000:00000010:0.1:1775770664.779329:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e610000. 00010000:00000010:0.1:1775770664.997111:0:180656:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e610c00. 00010000:00000010:0.1:1775770665.131200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345eda00. 00010000:00000010:0.1:1775770665.131210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e322800. 00010000:00000010:0.1:1775770665.213447:0:181478:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e610600. 00010000:00000010:0.1:1775770665.427165:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cca00. 00010000:00000010:0.1:1775770665.610772:0:181473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ed800. 00010000:00000010:0.1:1775770665.610781:0:181473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cdc00. 00010000:00000010:0.1:1775770665.631781:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cce00. 00010000:00000010:0.1:1775770665.639177:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a8c00. 00010000:00000010:0.1:1775770665.730752:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e186400. 00010000:00000010:0.1:1775770665.894354:0:171818:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cc200. 00010000:00000010:0.1:1775770666.092204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cc600. 00010000:00000010:0.1:1775770666.243714:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cd600. 00010000:00000010:0.1:1775770666.329288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e610c00. 00010000:00000010:0.1:1775770666.427331:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcfe00. 00010000:00000010:0.1:1775770666.636456:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201528800. 00010000:00000010:0.1:1775770666.836683:0:181437:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323e00. 00010000:00000010:0.1:1775770666.903601:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e611c00. 00010000:00000010:0.1:1775770666.903609:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e323a00. 00010000:00000010:0.1:1775770666.903620:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e610800. 00010000:00000010:0.1:1775770666.996252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6800. 00010000:00000010:0.1:1775770667.403252:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44400. 00010000:00000010:0.1:1775770667.527909:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6800. 00010000:00000010:0.1:1775770667.572828:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7400. 00010000:00000010:0.1:1775770667.776457:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6800. 00010000:00000010:0.1:1775770667.896683:0:181477:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6a00. 00010000:00000010:0.1:1775770667.985302:0:171818:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167f800. 00010000:00000010:0.1:1775770668.116717:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167ee00. 00010000:00000010:0.1:1775770668.241176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167fe00. 00010000:00000010:0.1:1775770668.443886:0:171818:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e611a00. 00010000:00000010:0.1:1775770668.655177:0:181473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ede00. 00010000:00000010:0.1:1775770668.729243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcfc00. 00010000:00000010:0.1:1775770668.837826:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da77800. 00010000:00000010:0.1:1775770669.021527:0:181472:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf71a00. 00010000:00000010:0.1:1775770669.078184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf71000. 00010000:00000010:0.1:1775770669.206557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf71c00. 00010000:00000010:0.1:1775770669.335864:0:181474:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf70a00. 00010000:00000010:0.1:1775770669.414305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec400. 00010000:00000010:0.1:1775770669.577028:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345eca00. 00010000:00000010:0.1:1775770669.577041:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a8200. 00010000:00000010:0.1:1775770669.634310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e610e00. 00010000:00000010:0.1:1775770669.722061:0:181474:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6800. 00010000:00000010:0.1:1775770669.740329:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf70e00. 00010000:00000010:0.1:1775770670.060262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73200. 00010000:00000010:0.1:1775770670.091240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec400. 00010000:00000010:0.1:1775770670.144642:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225863200. 00010000:00000010:0.1:1775770670.295785:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e611800. 00010000:00000010:0.1:1775770670.327516:0:181475:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72600. 00010000:00000010:0.1:1775770670.421153:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cde00. 00010000:00000010:0.1:1775770670.512001:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf71400. 00010000:00000010:0.1:1775770670.629163:0:171818:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf70200. 00010000:00000010:0.1:1775770670.806793:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72400. 00010000:00000010:0.1:1775770670.920969:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45600. 00010000:00000010:0.1:1775770670.920979:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec400. 00010000:00000010:0.1:1775770671.006622:0:181443:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72c00. 00010000:00000010:0.1:1775770671.063177:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a8600. 00010000:00000010:0.1:1775770671.277278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225862200. 00010000:00000010:0.1:1775770671.522715:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec200. 00010000:00000010:0.1:1775770671.674170:0:181474:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225863600. 00010000:00000010:0.1:1775770672.115061:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ed800. 00010000:00000010:0.1:1775770672.242575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345edc00. 00010000:00000010:0.1:1775770672.360238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bce000. 00010000:00000010:0.1:1775770672.428153:0:155798:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da77400. 00010000:00000010:0.1:1775770672.662886:0:181479:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7e00. 00010000:00000010:0.1:1775770673.026264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcfa00. 00010000:00000010:0.1:1775770673.112322:0:181479:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ede00. 00010000:00000010:0.1:1775770673.465167:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44400. 00010000:00000010:0.1:1775770673.646914:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6e00. 00010000:00000010:0.1:1775770673.914461:0:181472:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45600. 00010000:00000010:0.1:1775770674.214318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3800. 00010000:00000010:0.1:1775770674.369590:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dd000. 00010000:00000010:0.1:1775770674.369612:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2e00. 00010000:00000010:0.1:1775770674.566565:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44e00. 00010000:00000010:0.1:1775770674.767276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7000. 00010000:00000010:0.1:1775770674.988177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e610200. 00010000:00000010:0.1:1775770675.150387:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dc800. 00010000:00000010:0.1:1775770675.487160:0:181438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec600. 00010000:00000010:0.1:1775770675.487169:0:181438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a8c00. 00010000:00000010:0.1:1775770675.504175:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c893c00. 00010000:00000010:0.1:1775770675.507211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225863000. 00010000:00000010:0.1:1775770675.507224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e610600. 00010000:00000010:0.1:1775770675.603192:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6200. 00010000:00000010:0.1:1775770675.603212:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44400. 00010000:00000010:0.1:1775770675.603221:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c892400. 00010000:00000010:0.1:1775770675.831968:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa2400. 00010000:00000010:0.1:1775770676.071292:0:180656:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2e00. 00010000:00000010:0.1:1775770676.185277:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa3000. 00010000:00000010:0.1:1775770676.323685:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da76400. 00010000:00000010:0.1:1775770676.575216:0:181477:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dc800. 00010000:00000010:0.1:1775770676.656579:0:181477:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf71a00. 00010000:00000010:0.1:1775770676.821359:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ed200. 00010000:00000010:0.1:1775770676.941557:0:181441:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45e00. 00010000:00000010:0.1:1775770676.972953:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45a00. 00010000:00000010:0.1:1775770676.972959:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24600. 00010000:00000010:0.1:1775770676.972967:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3600. 00010000:00000010:0.1:1775770677.203222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c892c00. 00010000:00000010:0.1:1775770677.255196:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200030400. 00010000:00000010:0.1:1775770677.386200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb26e00. 00010000:00000010:0.1:1775770677.393270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb23a00. 00010000:00000010:0.1:1775770677.531442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24800. 00010000:00000010:0.1:1775770677.624540:0:181473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24e00. 00010000:00000010:0.1:1775770677.674189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa2c00. 00010000:00000010:0.1:1775770677.730239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3400. 00010000:00000010:0.1:1775770677.812179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200031000. 00010000:00000010:0.1:1775770677.882161:0:171818:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb27400. 00010000:00000010:0.1:1775770677.997893:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e611600. 00010000:00000010:0.1:1775770678.072513:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7c00. 00010000:00000010:0.1:1775770678.072533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb23600. 00010000:00000010:0.1:1775770678.072540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb26c00. 00010000:00000010:0.1:1775770678.143053:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa2200. 00010000:00000010:0.1:1775770678.153564:0:181475:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2400. 00010000:00000010:0.1:1775770678.153574:0:181475:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dc000. 00010000:00000010:0.1:1775770678.153602:0:181475:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3600. 00010000:00000010:0.1:1775770678.975443:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45800. 00010000:00000010:0.1:1775770679.186474:0:181474:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24400. 00010000:00000010:0.1:1775770679.186495:0:181474:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb26e00. 00010000:00000010:0.1:1775770679.191517:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25600. 00010000:00000010:0.1:1775770679.335275:0:155798:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a8400. 00010000:00000010:0.1:1775770679.364592:0:181477:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24200. 00010000:00000010:0.1:1775770679.530236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c892800. 00010000:00000010:0.1:1775770680.077205:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcee00. 00010000:00000010:0.1:1775770680.406162:0:155797:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72200. 00010000:00000010:0.1:1775770680.619170:0:181438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2200. 00010000:00000010:0.1:1775770680.636452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3e00. 00010000:00000010:0.1:1775770680.797720:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c893a00. 00010000:00000010:0.1:1775770681.099945:0:181474:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a9600. 00010000:00000010:0.1:1775770681.304572:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb26000. 00010000:00000010:0.1:1775770681.304578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a8400. 00010000:00000010:0.1:1775770681.466674:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2800. 00010000:00000010:0.1:1775770681.575534:0:181476:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73800. 00010000:00000010:0.1:1775770681.699495:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3c00. 00010000:00000010:0.1:1775770681.704319:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72c00. 00010000:00000010:0.1:1775770681.896477:0:155807:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345eca00. 00010000:00000010:0.1:1775770681.896482:0:155807:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb26e00. 00010000:00000010:0.1:1775770681.907993:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72a00. 00010000:00000010:0.1:1775770681.954950:0:181477:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf70000. 00010000:00000010:0.1:1775770682.036155:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e611600. 00010000:00000010:0.1:1775770682.036168:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb26000. 00010000:00000010:0.1:1775770682.043360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6a00. 00010000:00000010:0.1:1775770682.043364:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb27c00. 00010000:00000010:0.1:1775770682.049229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3400. 00010000:00000010:0.1:1775770682.131366:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa3600. 00010000:00000010:0.1:1775770682.166239:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2800. 00010000:00000010:0.1:1775770682.192825:0:181479:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206296a00. 00010000:00000010:0.1:1775770682.321206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ece00. 00010000:00000010:0.1:1775770682.329175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167f000. 00010000:00000010:0.1:1775770682.612805:0:181472:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72200. 00010000:00000010:0.1:1775770682.860157:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec400. 00010000:00000010:0.1:1775770682.940497:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a8200. 00010000:00000010:0.1:1775770683.022461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45400. 00010000:00000010:0.1:1775770683.103060:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3c00. 00010000:00000010:0.1:1775770683.123656:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190ca00. 00010000:00000010:0.1:1775770683.176186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ece00. 00010000:00000010:0.1:1775770683.180202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a8c00. 00010000:00000010:0.1:1775770683.180217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2200. 00010000:00000010:0.1:1775770683.327809:0:181477:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2400. 00010000:00000010:0.1:1775770683.403198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167e600. 00010000:00000010:0.1:1775770683.505192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190de00. 00010000:00000010:0.1:1775770683.509810:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2400. 00010000:00000010:0.1:1775770683.724201:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a9a00. 00010000:00000010:0.1:1775770683.773337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206296800. 00010000:00000010:0.1:1775770683.925790:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190d800. 00010000:00000010:0.1:1775770683.925801:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206297c00. 00010000:00000010:0.1:1775770683.994259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167fa00. 00010000:00000010:0.1:1775770684.110181:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ed800. 00010000:00000010:0.1:1775770684.162696:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcee00. 00010000:00000010:0.1:1775770684.162701:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcec00. 00010000:00000010:0.1:1775770684.168127:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec200. 00010000:00000010:0.1:1775770684.168162:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73600. 00010000:00000010:0.1:1775770684.168180:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206296400. 00010000:00000010:0.1:1775770684.243097:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6200. 00010000:00000010:0.1:1775770684.601355:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf71800. 00010000:00000010:0.1:1775770684.884161:0:155798:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dc400. 00010000:00000010:0.1:1775770684.897304:0:181479:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcee00. 00010000:00000010:0.1:1775770685.217160:0:181477:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25400. 00010000:00000010:0.1:1775770685.217169:0:181477:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200030800. 00010000:00000010:0.1:1775770685.294177:0:155798:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa3600. 00010000:00000010:0.1:1775770685.483650:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190c800. 00010000:00000010:0.1:1775770685.501582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3200. 00010000:00000010:0.1:1775770685.501600:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb23a00. 00010000:00000010:0.1:1775770685.515738:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec200. 00010000:00000010:0.1:1775770685.669925:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf70c00. 00010000:00000010:0.1:1775770685.748460:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44e00. 00010000:00000010:0.1:1775770686.019422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ecc00. 00010000:00000010:0.1:1775770686.097183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167fc00. 00010000:00000010:0.1:1775770686.125925:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6e00. 00010000:00000010:0.1:1775770686.184186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a9800. 00010000:00000010:0.1:1775770686.268320:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bce200. 00010000:00000010:0.1:1775770686.268325:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a8800. 00010000:00000010:0.1:1775770686.268328:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44200. 00010000:00000010:0.1:1775770686.268332:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45400. 00010000:00000010:0.1:1775770686.268340:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec600. 00010000:00000010:0.1:1775770686.282486:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167fe00. 00010000:00000010:0.1:1775770686.282494:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ed600. 00010000:00000010:0.1:1775770686.383181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3b7600. 00010000:00000010:0.1:1775770686.733164:0:181443:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e4a8000. 00010000:00000010:0.1:1775770686.733169:0:181443:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190cc00. 00010000:00000010:0.1:1775770686.745213:0:181472:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167f000. 00010000:00000010:0.1:1775770686.977578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3b7800. 00010000:00000010:0.1:1775770687.034573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa3c00. 00010000:00000010:0.1:1775770687.289599:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf71a00. 00010000:00000010:0.1:1775770687.475281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3b6600. 00010000:00000010:0.1:1775770687.538155:0:166678:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207c33800. 00010000:00000010:0.1:1775770687.561285:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72600. 00010000:00000010:0.1:1775770687.592184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6400. 00010000:00000010:0.1:1775770687.745159:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa3e00. 00010000:00000010:0.1:1775770687.846253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7800. 00010000:00000010:0.1:1775770687.869040:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207c32a00. 00010000:00000010:0.1:1775770687.993180:0:181477:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf70600. 00010000:00000010:0.1:1775770687.993188:0:181477:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207c32400. 00010000:00000010:0.1:1775770688.106273:0:181439:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3600. 00010000:00000010:0.1:1775770688.129823:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207c33400. 00010000:00000010:0.1:1775770688.417485:0:181479:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb23400. 00010000:00000010:0.1:1775770688.549448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb22000. 00010000:00000010:0.1:1775770688.620274:0:181439:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44a00. 00010000:00000010:0.1:1775770688.620288:0:181439:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf71e00. 00010000:00000010:0.1:1775770688.636259:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ed600. 00010000:00000010:0.1:1775770688.650353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcf000. 00010000:00000010:0.1:1775770688.650368:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcec00. 00010000:00000010:0.1:1775770688.650372:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf70400. 00010000:00000010:0.1:1775770688.676070:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3b6e00. 00010000:00000010:0.1:1775770688.822905:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf71e00. 00010000:00000010:0.1:1775770688.948246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bce000. 00010000:00000010:0.1:1775770689.009218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24000. 00010000:00000010:0.1:1775770689.139643:0:181472:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf70c00. 00010000:00000010:0.1:1775770689.332183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf71800. 00010000:00000010:0.1:1775770689.332190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44800. 00010000:00000010:0.1:1775770689.428316:0:181472:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25000. 00010000:00000010:0.1:1775770689.634623:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf71c00. 00010000:00000010:0.1:1775770689.718436:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcfc00. 00010000:00000010:0.1:1775770689.927966:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb27200. 00010000:00000010:0.1:1775770690.116382:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa3800. 00010000:00000010:0.1:1775770690.289158:0:181472:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf71800. 00010000:00000010:0.1:1775770690.586552:0:181472:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bce400. 00010000:00000010:0.1:1775770690.586560:0:181472:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24600. 00010000:00000010:0.1:1775770690.686336:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dc000. 00010000:00000010:0.1:1775770690.728580:0:181435:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25800. 00010000:00000010:0.1:1775770690.728585:0:181435:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa3e00. 00010000:00000010:0.1:1775770690.728587:0:181435:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcf000. 00010000:00000010:0.1:1775770690.736210:0:181435:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf71400. 00010000:00000010:0.1:1775770690.748900:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25e00. 00010000:00000010:0.1:1775770690.748923:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24400. 00010000:00000010:0.1:1775770690.951098:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72a00. 00010000:00000010:0.1:1775770691.062200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa3200. 00010000:00000010:0.1:1775770691.280334:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bce600. 00010000:00000010:0.1:1775770691.280351:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25200. 00010000:00000010:0.1:1775770691.359070:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72600. 00010000:00000010:0.1:1775770691.505469:0:155797:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201529600. 00010000:00000010:0.1:1775770691.505476:0:155797:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24000. 00010000:00000010:0.1:1775770691.515206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190d000. 00010000:00000010:0.1:1775770691.522177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190c800. 00010000:00000010:0.1:1775770691.660386:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25400. 00010000:00000010:0.1:1775770692.001075:0:181479:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24400. 00010000:00000010:0.1:1775770692.001092:0:181479:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73c00. 00010000:00000010:0.1:1775770692.657903:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25800. 00010000:00000010:0.1:1775770693.006278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bce600. 00010000:00000010:0.1:1775770693.097201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cde00. 00010000:00000010:0.1:1775770693.206008:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bf71a00. 00010000:00000010:0.1:1775770693.206014:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cdc00. 00010000:00000010:0.1:1775770693.213289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3c00. 00010000:00000010:0.1:1775770693.331480:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73a00. 00010000:00000010:0.1:1775770693.380214:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cd800. 00010000:00000010:0.1:1775770693.380219:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201529800. 00010000:00000010:0.1:1775770693.613970:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25e00. 00010000:00000010:0.1:1775770693.643650:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190d400. 00010000:00000010:0.1:1775770694.296429:0:181467:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ede00. 00010000:00000010:0.1:1775770695.537790:0:171818:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cc600. 00010000:00000010:0.1:1775770700.296192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec600. 00010000:00000010:0.1:1775770702.455104:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2600. 00010000:00000010:0.1:1775770702.641174:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3200. 00010000:00000010:0.1:1775770703.031177:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266ccc00. 00010000:00000010:0.1:1775770703.783850:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cce00. 00010000:00000010:0.1:1775770703.907316:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e800. 00010000:00000010:0.1:1775770703.907328:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2200. 00010000:00000010:0.1:1775770703.957379:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e200. 00010000:00000010:0.1:1775770703.957390:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3800. 00010000:00000010:0.1:1775770704.038201:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5900. 00010000:00000010:0.1:1775770704.038227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2400. 00010000:00000010:0.1:1775770704.060026:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cc800. 00010000:00000010:0.1:1775770704.131231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25200. 00010000:00000010:0.1:1775770704.404187:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5300. 00010000:00000010:0.1:1775770704.404196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2200. 00010000:00000010:0.1:1775770704.555101:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3400. 00010000:00000010:0.1:1775770704.555111:0:2404:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388eb00. 00010000:00000010:0.1:1775770704.555115:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2400. 00010000:00000010:0.1:1775770704.714243:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bed100. 00010000:00000010:0.1:1775770704.714256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cc800. 00010000:00000010:0.1:1775770704.781812:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5d00. 00010000:00000010:0.1:1775770704.781820:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3400. 00010000:00000010:0.1:1775770704.964198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190d000. 00010000:00000010:0.1:1775770704.964214:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9c200. 00010000:00000010:0.1:1775770704.964218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cdc00. 00010000:00000010:0.1:1775770705.109461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25800. 00010000:00000010:0.1:1775770705.185246:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cd400. 00010000:00000010:0.1:1775770705.367520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2800. 00010000:00000010:0.1:1775770705.367533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2600. 00010000:00000010:0.1:1775770705.893065:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9c800. 00010000:00000010:0.1:1775770705.893073:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3400. 00010000:00000010:0.1:1775770706.022482:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25e00. 00010000:00000010:0.1:1775770706.260386:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5e00. 00010000:00000010:0.1:1775770706.260396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24a00. 00010000:00000010:0.1:1775770706.351325:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233beda00. 00010000:00000010:0.1:1775770706.351341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cc400. 00010000:00000010:0.1:1775770706.513156:0:181473:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e500. 00010000:00000010:0.1:1775770706.513163:0:181473:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190c000. 00010000:00000010:0.1:1775770706.626039:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5200. 00010000:00000010:0.1:1775770706.626053:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2200. 00010000:00000010:0.1:1775770706.951227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2a00. 00010000:00000010:0.1:1775770706.978166:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5200. 00010000:00000010:0.1:1775770706.978186:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190de00. 00010000:00000010:0.1:1775770706.997545:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5a00. 00010000:00000010:0.1:1775770706.997560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190c000. 00010000:00000010:0.1:1775770707.011530:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3e00. 00010000:00000010:0.1:1775770707.024172:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5700. 00010000:00000010:0.1:1775770707.024180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2200. 00010000:00000010:0.1:1775770707.043248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c2400. 00010000:00000010:0.1:1775770707.083701:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c3200. 00010000:00000010:0.1:1775770707.223248:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bed500. 00010000:00000010:0.1:1775770707.223257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190ca00. 00010000:00000010:0.1:1775770707.343188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cda00. 00010000:00000010:0.1:1775770722.456189:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62266cce00. 00010000:00000010:0.1:1775770724.742227:0:181441:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da5f800. 00010000:00000010:0.1:1775770724.904464:0:181501:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190c400. 00010000:00000010:0.1:1775770726.350344:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5e00. 00010000:00000010:0.1:1775770726.350353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2e3600. 00010000:00000010:0.1:1775770739.960065:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25200. 00010000:00000010:0.1:1775770740.155317:0:181512:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dd000. 00000080:00000010:0.1:1775770740.234572:0:181512:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6205086300. 00000080:00000010:0.1:1775770740.400774:0:181512:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c6ccb00. 00010000:00000010:0.1:1775770740.469322:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25400. 00000080:00000010:0.1:1775770740.484316:0:181507:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c6ccf20. 00010000:00000010:0.1:1775770740.984174:0:181514:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6800. 00010000:00000010:0.1:1775770744.117447:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bedb00. 00010000:00000010:0.1:1775770744.117458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45800. 00010000:00000010:0.1:1775770744.117462:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bed300. 00010000:00000010:0.1:1775770744.117464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45c00. 00000080:00000010:0.1:1775770744.863392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62000256a0. 00010000:00000010:0.1:1775770889.443673:0:182749:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225aa8000. 00010000:00000010:0.1:1775770908.231199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e08da00. 00010000:00000010:0.1:1775770908.231210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190d200. 00010000:00000010:0.1:1775770908.231215:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e800. 00010000:00000010:0.1:1775770908.231219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190d000. 00010000:00000010:0.1:1775770908.231238:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6400. 00010000:00000010:0.1:1775770908.231241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233783000. 00000080:00000010:0.1:1775770908.287304:0:182753:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd841c0. 00000080:00000010:0.1:1775770911.669554:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621906ca40. 00010000:00000010:0.1:1775770913.440436:0:2404:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e300. 00010000:00000010:0.1:1775770913.440445:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190de00. 00010000:00000010:0.1:1775770913.440451:0:2404:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6600. 00010000:00000010:0.1:1775770913.440457:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44a00. 00010000:00000010:0.1:1775770913.440462:0:2404:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6300. 00010000:00000010:0.1:1775770913.440469:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45600. 00000080:00000010:0.1:1775770920.283319:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd460. 00010000:00000010:0.1:1775770920.307195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190ce00. 00000080:00000010:0.1:1775770946.439162:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62190698c0. 00010000:00000010:0.1:1775770949.209130:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190c200. 00010000:00000010:0.1:1775770950.044021:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9c100. 00010000:00000010:0.1:1775770950.044030:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24400. 00000080:00000010:0.1:1775770987.586222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd280. 00010000:00000010:0.1:1775771017.076239:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6700. 00010000:00000010:0.1:1775771017.076246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7800. 00010000:00000010:0.1:1775771017.176207:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9c700. 00010000:00000010:0.1:1775771017.176214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73a00. 00010000:00000010:0.1:1775771017.427183:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5200. 00010000:00000010:0.1:1775771017.427194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206295a00. 00010000:00000010:0.1:1775771026.307297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72c00. 00010000:00000010:0.1:1775771026.307310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24400. 00010000:00000010:0.1:1775771026.307312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72600. 00010000:00000010:0.1:1775771026.307317:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5e00. 00010000:00000010:0.1:1775771026.307321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206294e00. 00010000:00000010:0.1:1775771026.307339:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206551300. 00010000:00000010:0.1:1775771026.307342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73e00. 00000080:00000010:0.1:1775771026.319190:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621906a100. 00010000:00000010:0.1:1775771028.197209:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5300. 00010000:00000010:0.1:1775771028.197341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6e00. 00010000:00000010:0.1:1775771028.211163:0:166678:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5400. 00010000:00000010:0.1:1775771028.232607:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5d00. 00010000:00000010:0.1:1775771028.232633:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6e00. 00000080:00000010:0.1:1775771028.284208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84280. 00010000:00000010:0.1:1775771028.308501:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5400. 00010000:00000010:0.1:1775771028.308514:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7600. 00010000:00000010:0.1:1775771028.327738:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5e00. 00010000:00000010:0.1:1775771028.448252:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5e00. 00010000:00000010:0.1:1775771028.448263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7800. 00010000:00000010:0.1:1775771028.457961:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5f00. 00010000:00000010:0.1:1775771028.457974:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6a00. 00000800:00000200:0.1:1775771029.795911:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935100 low water 1629602 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775771029.795925:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935100 low water 1629602 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775771029.795934:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935100 low water 1629602 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775771029.795945:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935100 low water 1629602 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775771029.795955:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935100 low water 1629602 conn ffff8f62079ae000 blocked idle queued 00000080:00000010:0.1:1775771058.789598:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f027380. 00010000:00000010:0.1:1775771093.676206:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5400. 00010000:00000010:0.1:1775771093.676226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6209e73200. 00010000:00000010:0.1:1775771093.676230:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5c00. 00010000:00000010:0.1:1775771096.364291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24200. 00010000:00000010:0.1:1775771096.364304:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25600. 00010000:00000010:0.1:1775771096.364309:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6200. 00010000:00000010:0.1:1775771096.364313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6200. 00010000:00000010:0.1:1775771096.364317:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9c600. 00010000:00000010:0.1:1775771096.364319:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25a00. 00000080:00000010:0.1:1775771096.392186:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f027380. 00010000:00000010:0.1:1775771171.143993:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25c00. 00010000:00000010:0.1:1775771171.576292:0:188483:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6f00. 00010000:00000010:0.1:1775771171.576329:0:188483:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622586cc00. 00000080:00000010:0.1:1775771290.505089:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c6ccf40. 00010000:00000010:0.1:1775771292.563727:0:188541:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9cf00. 00010000:00000010:0.1:1775771292.563735:0:188541:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25600. 00000080:00000010:0.1:1775771295.857202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62000256e0. 00000080:00000010:0.1:1775771353.930332:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6200025c80. 00010000:00000010:0.1:1775771367.564319:0:189166:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388ea00. 00010000:00000010:0.1:1775771367.564337:0:189166:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72e00. 00010000:00000010:0.1:1775771367.590429:0:189166:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e300. 00010000:00000010:0.1:1775771367.590446:0:189166:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73800. 00000080:00000010:0.1:1775771403.424263:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f620216a100. 00010000:00000010:0.1:1775771405.037196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6e00. 00010000:00000010:0.1:1775771415.901120:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6a00. 00010000:00000010:0.1:1775771415.901163:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210405200. 00010000:00000010:0.1:1775771417.733165:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45e00. 00010000:00000010:0.1:1775771420.261301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72a00. 00010000:00000010:0.1:1775771423.950438:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6fa00. 00010000:00000010:0.1:1775771423.950443:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2dc600. 00010000:00000010:0.1:1775771435.474251:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6fa00. 00010000:00000010:0.1:1775771435.474261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62202c9600. 00010000:00000010:0.1:1775771463.003427:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24e00. 00010000:00000010:0.1:1775771463.026110:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234896000. 00010000:00000010:0.1:1775771463.026117:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234897000. 00010000:00000010:0.1:1775771463.026120:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234896e00. 00010000:00000010:0.1:1775771463.026123:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234896a00. 00010000:00000010:0.1:1775771463.026125:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234896400. 00010000:00000010:0.1:1775771463.026128:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234896200. 00010000:00000010:0.1:1775771463.026130:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234896800. 00010000:00000010:0.1:1775771463.038369:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0fe00. 00010000:00000010:0.1:1775771463.038376:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0f000. 00010000:00000010:0.1:1775771463.038378:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0f600. 00010000:00000010:0.1:1775771463.038381:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0fc00. 00010000:00000010:0.1:1775771463.038383:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0fa00. 00010000:00000010:0.1:1775771463.038463:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0f400. 00010000:00000010:0.1:1775771463.038469:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0ee00. 00010000:00000010:0.1:1775771463.053257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0e000. 00010000:00000010:0.1:1775771463.401102:0:190357:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0e200. 00010000:00000010:0.1:1775771464.236937:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73800. 00010000:00000010:0.1:1775771465.632206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24800. 00010000:00000010:0.1:1775771470.120215:0:180656:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73a00. 00010000:00000010:0.1:1775771477.695208:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9cf00. 00010000:00000010:0.1:1775771477.695221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234896e00. 00010000:00000010:0.1:1775771477.761806:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9c000. 00010000:00000010:0.1:1775771477.761822:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25600. 00010000:00000010:0.1:1775771483.691417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190dc00. 00010000:00000010:0.1:1775771484.190272:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6c00. 00010000:00000010:0.1:1775771484.190285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621190c800. 00010000:00000010:0.1:1775771484.190289:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6fe00. 00010000:00000010:0.1:1775771484.218739:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6fd00. 00010000:00000010:0.1:1775771484.218755:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0fa00. 00000080:00000010:0.1:1775771484.258176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84580. 00000080:00000010:0.1:1775771495.153502:0:8206:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226bc5ac0. 00010000:00000010:0.1:1775771497.176224:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9c900. 00010000:00000010:0.1:1775771497.176231:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ed000. 00010000:00000010:0.1:1775771497.205044:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9cc00. 00010000:00000010:0.1:1775771497.205059:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25400. 00000080:00000010:0.1:1775771497.244196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdba60. 00010000:00000010:0.1:1775771498.098988:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388eb00. 00010000:00000010:0.1:1775771498.098997:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ede00. 00010000:00000010:0.1:1775771498.744185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623342ee00. 00010000:00000010:0.1:1775771499.166203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec600. 00010000:00000010:0.1:1775771535.068110:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388ee00. 00010000:00000010:0.1:1775771535.068120:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345edc00. 00010000:00000010:0.1:1775771595.734475:0:2404:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6fd00. 00010000:00000010:0.1:1775771595.734486:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201529400. 00010000:00000010:0.1:1775771596.314304:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201528a00. 00010000:00000010:0.1:1775771597.429510:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec600. 00000800:00000200:0.1:1775771599.180336:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1318084 low water 1438110 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775771599.180347:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1318084 low water 1438110 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775771599.180357:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1318084 low water 1438110 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775771599.180368:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1318084 low water 1438110 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775771599.180377:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1318084 low water 1438110 conn ffff8f62079ae000 blocked idle queued 00010000:00000010:0.1:1775771607.980946:0:12:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6f700. 00010000:00000010:0.1:1775771607.980959:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210405200. 00010000:00000010:0.1:1775771631.602187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672e400. 00010000:00000010:0.1:1775771631.602194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672ec00. 00010000:00000010:0.1:1775771631.602196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672ee00. 00010000:00000010:0.1:1775771631.602198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672fa00. 00010000:00000010:0.1:1775771631.602200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672f600. 00010000:00000010:0.1:1775771631.602202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672ea00. 00010000:00000010:0.1:1775771631.602203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672e800. 00010000:00000010:0.1:1775771646.461633:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec400. 00010000:00000010:0.1:1775771646.461639:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25e00. 00010000:00000010:0.1:1775771646.464213:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e900. 00010000:00000010:0.1:1775771646.464221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ed200. 00010000:00000010:0.1:1775771646.464225:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6000. 00010000:00000010:0.1:1775771646.464228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45e00. 00010000:00000010:0.1:1775771652.795160:0:191552:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e000. 00010000:00000010:0.1:1775771652.795169:0:191552:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345edc00. 00010000:00000010:0.1:1775771653.283371:0:141703:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6300. 00010000:00000010:0.1:1775771653.283381:0:141703:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44400. 00000080:00000010:0.1:1775771661.767207:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202168000. 00010000:00000010:0.1:1775771720.721497:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9c100. 00010000:00000010:0.1:1775771720.721512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25400. 00010000:00000010:0.1:1775771720.721516:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24400. 00010000:00000010:0.1:1775771720.721535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25c00. 00000080:00000010:0.1:1775771721.096178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62050869e0. 00010000:00000010:0.1:1775771721.122163:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9c300. 00010000:00000010:0.1:1775771721.122177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25600. 00010000:00000010:0.1:1775771721.122181:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e200. 00010000:00000010:0.1:1775771721.122184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ecc00. 00010000:00000010:0.1:1775771836.515254:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6e00. 00010000:00000010:0.1:1775771836.515266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fce9c00. 00010000:00000010:0.1:1775771836.515270:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6000. 00010000:00000010:0.1:1775771836.515273:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101126600. 00010000:00000010:0.1:1775771836.544192:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6200. 00010000:00000010:0.1:1775771836.544202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101126000. 00010000:00000010:0.1:1775771880.790440:0:193527:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e300. 00010000:00000010:0.1:1775771880.790455:0:193527:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ed800. 00000080:00000010:0.1:1775772107.113168:0:202410:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f620991e300. 00000080:00000010:0.1:1775772107.113184:0:202410:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62099198c0. 00000080:00000010:0.1:1775772107.113190:0:202410:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f620991a100. 00000080:00000010:0.1:1775772107.113198:0:202410:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f620991c200. 00000080:00000010:0.1:1775772107.113204:0:202410:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209918000. 00000080:00000010:0.1:1775772107.113208:0:202410:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f620991eb40. 00000080:00000010:0.1:1775772107.113212:0:202410:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209919080. 00000080:00000010:0.1:1775772107.113218:0:202410:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185b8840. 00000080:00000010:0.1:1775772107.113224:0:202410:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f620991dac0. 00000080:00000010:0.1:1775772107.113228:0:202410:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021bd280. 00000080:00000010:0.1:1775772107.113235:0:202410:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021bb180. 00000080:00000010:0.1:1775772107.113239:0:202410:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185bc200. 00000080:00000010:0.1:1775772107.125206:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021ba940. 00000080:00000010:0.1:1775772107.125228:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621edac200. 00000080:00000010:0.1:1775772107.125235:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021bca40. 00000080:00000010:0.1:1775772107.125243:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185ba940. 00000080:00000010:0.1:1775772107.125260:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621904a100. 00000080:00000010:0.1:1775772107.125295:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021be300. 00000080:00000010:0.1:1775772107.125301:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021b98c0. 00000080:00000010:0.1:1775772107.125309:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021ba100. 00000080:00000010:0.1:1775772107.125314:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021bc200. 00000080:00000010:0.1:1775772107.125322:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021b8000. 00000080:00000010:0.1:1775772107.125326:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021beb40. 00000080:00000010:0.1:1775772107.125334:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621904a940. 00000080:00000010:0.1:1775772107.125340:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62190498c0. 00000080:00000010:0.1:1775772107.125345:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621904eb40. 00000080:00000010:0.1:1775772107.125351:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621904e300. 00000080:00000010:0.1:1775772107.125356:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021ff380. 00000080:00000010:0.1:1775772107.125362:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021fe300. 00000080:00000010:0.1:1775772107.125368:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621eda8840. 00000080:00000010:0.1:1775772107.125382:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621edaf380. 00000080:00000010:0.1:1775772107.125389:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021b9080. 00000080:00000010:0.1:1775772107.125394:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021bdac0. 00000080:00000010:0.1:1775772107.125399:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021bf380. 00000080:00000010:0.1:1775772107.125404:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021b8840. 00000080:00000010:0.1:1775772107.125408:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021bb9c0. 00000080:00000010:0.1:1775772107.125414:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6220ae2100. 00000080:00000010:0.1:1775772107.125419:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621eda9080. 00000080:00000010:0.1:1775772107.125424:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621edaa100. 00000080:00000010:0.1:1775772107.125430:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621edaa940. 00000080:00000010:0.1:1775772107.125436:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621eda3180. 00000080:00000010:0.1:1775772107.125442:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621eda2940. 00000080:00000010:0.1:1775772107.125448:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621eda4a40. 00000080:00000010:0.1:1775772107.125454:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6220ae4200. 00000080:00000010:0.1:1775772107.125459:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6220ae0000. 00000080:00000010:0.1:1775772107.125464:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6220ae6b40. 00000080:00000010:0.1:1775772107.125470:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6220ae1080. 00000080:00000010:0.1:1775772107.125475:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6220ae5ac0. 00000080:00000010:0.1:1775772107.125486:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6220ae7380. 00000080:00000010:0.1:1775772107.125491:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6220ae0840. 00000080:00000010:0.1:1775772107.125496:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6220ae39c0. 00000080:00000010:0.1:1775772107.125500:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6220ae5280. 00000080:00000010:0.1:1775772107.125509:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6220ae3180. 00000080:00000010:0.1:1775772107.125514:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6220ae2940. 00000080:00000010:0.1:1775772107.125519:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021fb180. 00000080:00000010:0.1:1775772107.125523:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021fa100. 00000080:00000010:0.1:1775772107.125527:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021fb9c0. 00000080:00000010:0.1:1775772107.125532:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021f9080. 00000080:00000010:0.1:1775772107.125536:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021fd280. 00000080:00000010:0.1:1775772107.125539:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021fc200. 00000080:00000010:0.1:1775772107.125544:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021fca40. 00000080:00000010:0.1:1775772107.125547:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021f8840. 00000080:00000010:0.1:1775772107.125551:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021feb40. 00000080:00000010:0.1:1775772107.125555:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021fdac0. 00000080:00000010:0.1:1775772107.125559:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021f8000. 00000080:00000010:0.1:1775772107.125563:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021f98c0. 00000080:00000010:0.1:1775772107.125566:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021fa940. 00000080:00000010:0.1:1775772107.125571:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62269fc200. 00000080:00000010:0.1:1775772107.125576:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62269f9080. 00000080:00000010:0.1:1775772107.125580:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62269fdac0. 00000080:00000010:0.1:1775772107.125585:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62269f98c0. 00000080:00000010:0.1:1775772107.125589:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62269fb180. 00000080:00000010:0.1:1775772107.125593:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62269f8840. 00000080:00000010:0.1:1775772107.125598:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62269f8000. 00000080:00000010:0.1:1775772107.125602:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62269fca40. 00000080:00000010:0.1:1775772107.125605:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62269fd280. 00000080:00000010:0.1:1775772107.125609:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62269fe300. 00000080:00000010:0.1:1775772107.125613:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62269fb9c0. 00000080:00000010:0.1:1775772107.125616:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62269feb40. 00000080:00000010:0.1:1775772107.125628:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62269ff380. 00000080:00000010:0.1:1775772107.125632:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62269fa940. 00000080:00000010:0.1:1775772107.125637:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621eda6300. 00000080:00000010:0.1:1775772107.125643:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621eda18c0. 00000080:00000010:0.1:1775772107.125647:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621eda2100. 00000080:00000010:0.1:1775772107.125650:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621eda4200. 00000080:00000010:0.1:1775772107.125654:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621eda0000. 00000080:00000010:0.1:1775772107.125657:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621eda6b40. 00000080:00000010:0.1:1775772107.125661:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621eda1080. 00000080:00000010:0.1:1775772107.125666:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621eda5ac0. 00000080:00000010:0.1:1775772107.125669:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621eda7380. 00000080:00000010:0.1:1775772107.125672:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621eda0840. 00000080:00000010:0.1:1775772107.125676:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621eda39c0. 00000080:00000010:0.1:1775772107.125679:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621eda5280. 00000080:00000010:0.1:1775772107.125682:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020e3180. 00000080:00000010:0.1:1775772107.125721:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020e2940. 00000080:00000010:0.1:1775772107.125726:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6220ae4a40. 00000080:00000010:0.1:1775772107.125732:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6220ae6300. 00000080:00000010:0.1:1775772107.125735:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6220ae18c0. 00000080:00000010:0.1:1775772107.125745:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202274a40. 00000080:00000010:0.1:1775772107.125750:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202276300. 00000080:00000010:0.1:1775772107.125754:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62022718c0. 00000080:00000010:0.1:1775772107.125758:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202272100. 00000080:00000010:0.1:1775772107.125762:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202274200. 00000080:00000010:0.1:1775772107.125767:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202270000. 00000080:00000010:0.1:1775772107.125782:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202276b40. 00000080:00000010:0.1:1775772107.125787:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202271080. 00000080:00000010:0.1:1775772107.125791:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202275ac0. 00000080:00000010:0.1:1775772107.125795:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202277380. 00000080:00000010:0.1:1775772107.125800:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202270840. 00000080:00000010:0.1:1775772107.125804:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62022739c0. 00000080:00000010:0.1:1775772107.125816:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202275280. 00000080:00000010:0.1:1775772107.125820:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202273180. 00000080:00000010:0.1:1775772107.125823:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202272940. 00000080:00000010:0.1:1775772107.125827:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62095d5ac0. 00000080:00000010:0.1:1775772107.125833:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62095d7380. 00000080:00000010:0.1:1775772107.125837:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62095d0840. 00000080:00000010:0.1:1775772107.125842:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62095d39c0. 00000080:00000010:0.1:1775772107.125847:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020e4a40. 00000080:00000010:0.1:1775772107.125852:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020e6300. 00000080:00000010:0.1:1775772107.125856:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020e18c0. 00000080:00000010:0.1:1775772107.125860:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020e2100. 00000080:00000010:0.1:1775772107.125865:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020e4200. 00000080:00000010:0.1:1775772107.125869:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62095d5280. 00000080:00000010:0.1:1775772107.125873:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62095d3180. 00000080:00000010:0.1:1775772107.125877:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62095d2940. 00000080:00000010:0.1:1775772107.125882:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62095d4a40. 00000080:00000010:0.1:1775772107.125886:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62095d6300. 00000080:00000010:0.1:1775772107.125890:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020e0000. 00000080:00000010:0.1:1775772107.125894:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020e6b40. 00000080:00000010:0.1:1775772107.125897:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020e1080. 00000080:00000010:0.1:1775772107.125900:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62095d18c0. 00000080:00000010:0.1:1775772107.125904:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62095d2100. 00000080:00000010:0.1:1775772107.125908:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62095d4200. 00000080:00000010:0.1:1775772107.125911:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62095d0000. 00000080:00000010:0.1:1775772107.125915:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62095d6b40. 00000080:00000010:0.1:1775772107.125919:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020e5ac0. 00000080:00000010:0.1:1775772107.125923:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020e7380. 00000080:00000010:0.1:1775772107.125928:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62095d1080. 00000080:00000010:0.1:1775772107.125932:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021f6300. 00000080:00000010:0.1:1775772107.125936:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021f18c0. 00000080:00000010:0.1:1775772107.125943:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021f2100. 00000080:00000010:0.1:1775772107.125955:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021f4200. 00000080:00000010:0.1:1775772107.125960:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021f0000. 00000080:00000010:0.1:1775772107.125964:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021f6b40. 00000080:00000010:0.1:1775772107.125967:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021f1080. 00000080:00000010:0.1:1775772107.125971:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021f5ac0. 00000080:00000010:0.1:1775772107.125975:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021f7380. 00000080:00000010:0.1:1775772107.125978:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021f0840. 00000080:00000010:0.1:1775772107.125983:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021f39c0. 00000080:00000010:0.1:1775772107.125986:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021f5280. 00000080:00000010:0.1:1775772107.125990:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021f3180. 00000080:00000010:0.1:1775772107.125993:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021f2940. 00000080:00000010:0.1:1775772107.125997:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62021f4a40. 00000080:00000010:0.1:1775772107.126000:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020e0840. 00000080:00000010:0.1:1775772107.126004:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6219049080. 00000080:00000010:0.1:1775772107.126018:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020f98c0. 00000080:00000010:0.1:1775772107.126025:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62269fa100. 00000080:00000010:0.1:1775772107.126037:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020fa100. 00000080:00000010:0.1:1775772107.126041:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185bb9c0. 00000080:00000010:0.1:1775772107.126046:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020e39c0. 00000080:00000010:0.1:1775772107.126051:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226afdac0. 00000080:00000010:0.1:1775772107.126058:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020fc200. 00000080:00000010:0.1:1775772107.126062:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020f8000. 00000080:00000010:0.1:1775772107.126072:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020e5280. 00000080:00000010:0.1:1775772107.126076:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020fe300. 00000080:00000010:0.1:1775772107.126081:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020feb40. 00000080:00000010:0.1:1775772107.126086:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185a39c0. 00000080:00000010:0.1:1775772107.126091:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020f9080. 00000080:00000010:0.1:1775772107.126095:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020fdac0. 00000080:00000010:0.1:1775772107.126099:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020ff380. 00000080:00000010:0.1:1775772107.126103:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020f8840. 00000080:00000010:0.1:1775772107.126107:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020fb9c0. 00000080:00000010:0.1:1775772107.126118:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020fd280. 00000080:00000010:0.1:1775772107.126123:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020fb180. 00000080:00000010:0.1:1775772107.126126:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020fa940. 00000080:00000010:0.1:1775772107.126130:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62020fca40. 00000080:00000010:0.1:1775772107.126160:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226826b40. 00000080:00000010:0.1:1775772107.126190:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226821080. 00000080:00000010:0.1:1775772107.126196:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226825ac0. 00000080:00000010:0.1:1775772107.126202:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621efeeb40. 00000080:00000010:0.1:1775772107.126210:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226af9080. 00000080:00000010:0.1:1775772107.126214:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226afd280. 00000080:00000010:0.1:1775772107.126219:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226afa940. 00000080:00000010:0.1:1775772107.126225:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226afe300. 00000080:00000010:0.1:1775772107.126229:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226af98c0. 00000080:00000010:0.1:1775772107.126234:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226827380. 00000080:00000010:0.1:1775772107.126239:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226820840. 00000080:00000010:0.1:1775772107.126243:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62268239c0. 00000080:00000010:0.1:1775772107.126249:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226825280. 00000080:00000010:0.1:1775772107.126253:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226823180. 00000080:00000010:0.1:1775772107.126258:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226822940. 00000080:00000010:0.1:1775772107.126263:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226824a40. 00000080:00000010:0.1:1775772107.126268:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226826300. 00000080:00000010:0.1:1775772107.126273:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226af8000. 00000080:00000010:0.1:1775772107.126278:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226afa100. 00000080:00000010:0.1:1775772107.126295:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226af8840. 00000080:00000010:0.1:1775772107.126301:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226afc200. 00000080:00000010:0.1:1775772107.126306:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226afb9c0. 00000080:00000010:0.1:1775772107.126311:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226afeb40. 00000080:00000010:0.1:1775772107.126316:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226afca40. 00000080:00000010:0.1:1775772107.126326:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226aff380. 00000080:00000010:0.1:1775772107.126333:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62268218c0. 00000080:00000010:0.1:1775772107.126339:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226822100. 00000080:00000010:0.1:1775772107.126350:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226824200. 00000080:00000010:0.1:1775772107.126355:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226820000. 00000080:00000010:0.1:1775772107.126360:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62022a3180. 00000080:00000010:0.1:1775772107.126365:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62022a2940. 00000080:00000010:0.1:1775772107.126370:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62022a4a40. 00000080:00000010:0.1:1775772107.126376:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62022a6300. 00000080:00000010:0.1:1775772107.126386:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62022a18c0. 00000080:00000010:0.1:1775772107.126391:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62022a2100. 00000080:00000010:0.1:1775772107.126399:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62022a4200. 00000080:00000010:0.1:1775772107.132242:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62022a0000. 00000080:00000010:0.1:1775772107.132262:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62022a6b40. 00000080:00000010:0.1:1775772107.132271:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62022a1080. 00000080:00000010:0.1:1775772107.132565:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62022a5ac0. 00000080:00000010:0.1:1775772107.132576:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62022a7380. 00000080:00000010:0.1:1775772107.132581:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62022a0840. 00000080:00000010:0.1:1775772107.132586:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62022a39c0. 00000080:00000010:0.1:1775772107.132591:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62022a5280. 00000080:00000010:0.1:1775772107.132597:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209aeb180. 00000080:00000010:0.1:1775772107.132603:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209aea940. 00000080:00000010:0.1:1775772107.132613:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209aeca40. 00000080:00000010:0.1:1775772107.132624:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185a5280. 00000080:00000010:0.1:1775772107.132630:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185a3180. 00000080:00000010:0.1:1775772107.132635:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185a2940. 00000080:00000010:0.1:1775772107.132642:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209aee300. 00000080:00000010:0.1:1775772107.132647:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209ae98c0. 00000080:00000010:0.1:1775772107.132653:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209aea100. 00000080:00000010:0.1:1775772107.132657:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209aec200. 00000080:00000010:0.1:1775772107.132662:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209ae8000. 00000080:00000010:0.1:1775772107.132667:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209aeeb40. 00000080:00000010:0.1:1775772107.132770:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209ae9080. 00000080:00000010:0.1:1775772107.132778:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209aedac0. 00000080:00000010:0.1:1775772107.132797:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209aef380. 00000080:00000010:0.1:1775772107.132802:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209ae8840. 00000080:00000010:0.1:1775772107.132807:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209aeb9c0. 00000080:00000010:0.1:1775772107.132812:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209aed280. 00000080:00000010:0.1:1775772107.132817:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185a4a40. 00000080:00000010:0.1:1775772107.132822:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185a6300. 00000080:00000010:0.1:1775772107.132828:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185a18c0. 00000080:00000010:0.1:1775772107.132833:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192da940. 00000080:00000010:0.1:1775772107.132840:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192dca40. 00000080:00000010:0.1:1775772107.132847:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192de300. 00000080:00000010:0.1:1775772107.132852:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192d98c0. 00000080:00000010:0.1:1775772107.132856:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192da100. 00000080:00000010:0.1:1775772107.132862:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192dc200. 00000080:00000010:0.1:1775772107.132869:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621efeb9c0. 00000080:00000010:0.1:1775772107.132875:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192d8000. 00010000:00000010:0.1:1775772107.297939:0:149312:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45000. 00010000:00000010:0.1:1775772107.297947:0:149312:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233926400. 00010000:00000010:0.1:1775772118.468158:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ed600. 00010000:00000010:0.1:1775772118.468165:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388ef00. 00010000:00000010:0.1:1775772118.468170:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ece00. 00010000:00000010:0.1:1775772118.468173:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9cc00. 00010000:00000010:0.1:1775772118.468176:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24000. 00010000:00000010:0.1:1775772137.834310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45c00. 00010000:00000010:0.1:1775772138.839319:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45c00. 00000080:00000010:0.1:1775772138.839340:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792c760. 00010000:00000010:0.1:1775772138.876928:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45800. 00000080:00000010:0.1:1775772138.876943:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792ce40. 00010000:00000010:0.1:1775772145.291791:0:204930:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44600. 00010000:00000010:0.1:1775772145.321475:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45200. 00010000:00000010:0.1:1775772145.451409:0:204934:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44600. 00010000:00000010:0.1:1775772146.107366:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e300. 00010000:00000010:0.1:1775772146.107382:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ed400. 00010000:00000010:0.1:1775772146.107418:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e400. 00010000:00000010:0.1:1775772146.107420:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ed000. 00010000:00000010:0.1:1775772146.135723:0:204950:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9c000. 00010000:00000010:0.1:1775772146.135733:0:204950:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ede00. 00010000:00000010:0.1:1775772146.135738:0:204950:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6b00. 00010000:00000010:0.1:1775772146.135740:0:204950:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101126800. 00000080:00000010:0.1:1775772146.153186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2060. 00000080:00000010:0.1:1775772146.160291:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f620216b9c0. 00010000:00000010:0.1:1775772178.877789:0:2404:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6f900. 00010000:00000010:0.1:1775772178.877804:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44400. 00000080:00000010:0.1:1775772179.914018:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2900. 00000080:00000010:0.1:1775772179.954208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4ae0. 00010000:00000010:0.1:1775772179.989184:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6fa00. 00010000:00000010:0.1:1775772179.989190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45a00. 00000080:00000010:0.1:1775772180.057259:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2da0. 00010000:00000010:0.1:1775772180.441648:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45a00. 00010000:00000010:0.1:1775772181.723958:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5400. 00010000:00000010:0.1:1775772181.723980:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44400. 00010000:00000010:0.1:1775772182.073161:0:206307:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5a00. 00010000:00000010:0.1:1775772183.408791:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c25f600. 00010000:00000010:0.1:1775772183.408799:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101123800. 00010000:00000010:0.1:1775772183.426204:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5a00. 00010000:00000010:0.1:1775772183.449530:0:206307:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6c00. 00000080:00000010:0.1:1775772183.459209:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4e00. 00000080:00000010:0.1:1775772183.900238:0:206307:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fda00. 00000080:00000010:0.1:1775772183.926297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4da0. 00010000:00000010:0.1:1775772183.937114:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6c00. 00000080:00000010:0.1:1775772183.947282:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4e00. 00000080:00000010:0.1:1775772183.962350:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4e00. 00010000:00000010:0.1:1775772183.993184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7000. 00010000:00000010:0.1:1775772184.766016:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5900. 00000080:00000010:0.1:1775772185.461332:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd7e0. 00000080:00000010:0.1:1775772199.405189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fdc0. 00010000:00000010:0.1:1775772199.548170:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202ae7f00. 00010000:00000010:0.1:1775772199.548183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00000080:00000010:0.1:1775772199.548187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fb60. 00010000:00000010:0.1:1775772199.789389:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6100. 00010000:00000010:0.1:1775772199.789403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00000080:00000010:0.1:1775772199.890220:0:206462:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fc40. 00010000:00000010:0.1:1775772200.173278:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6e00. 00010000:00000010:0.1:1775772200.173290:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5900. 00010000:00000010:0.1:1775772200.173294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00000080:00000010:0.1:1775772200.190175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f1c0. 00010000:00000010:0.1:1775772200.611759:0:2404:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5900. 00010000:00000010:0.1:1775772200.611778:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89e00. 00000080:00000010:0.1:1775772200.618191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f0e0. 00010000:00000010:0.1:1775772200.797221:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5900. 00010000:00000010:0.1:1775772200.797307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88a00. 00000080:00000010:0.1:1775772200.797313:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fe20. 00010000:00000010:0.1:1775772200.914550:0:206728:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6e00. 00010000:00000010:0.1:1775772200.914570:0:206728:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00000080:00000010:0.1:1775772200.914577:0:206728:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f4a0. 00010000:00000010:0.1:1775772200.940391:0:206728:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5900. 00010000:00000010:0.1:1775772200.940409:0:206728:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89800. 00000080:00000010:0.1:1775772200.940416:0:206728:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f000. 00010000:00000010:0.1:1775772200.995963:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5900. 00010000:00000010:0.1:1775772200.995980:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120ca500. 00010000:00000010:0.1:1775772200.995985:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89800. 00010000:00000010:0.1:1775772201.167205:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120ca300. 00010000:00000010:0.1:1775772201.167221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88600. 00000080:00000010:0.1:1775772201.167225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f2e0. 00010000:00000010:0.1:1775772201.236776:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bedb00. 00010000:00000010:0.1:1775772201.236790:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226004a00. 00000080:00000010:0.1:1775772201.236796:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0ef60. 00010000:00000010:0.1:1775772202.217815:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202ae7600. 00010000:00000010:0.1:1775772202.217831:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:0.1:1775772203.212849:0:13:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120caa00. 00010000:00000010:0.1:1775772203.212861:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88000. 00000080:00000010:0.1:1775772203.212866:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fa00. 00010000:00000010:0.1:1775772203.239179:0:2404:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64d00. 00010000:00000010:0.1:1775772203.239188:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00000080:00000010:0.1:1775772203.275281:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371ff00. 00010000:00000010:0.1:1775772205.030412:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bed000. 00010000:00000010:0.1:1775772205.030432:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120ca200. 00010000:00000010:0.1:1775772205.030436:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226005200. 00000080:00000010:0.1:1775772205.108486:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0efa0. 00010000:00000010:0.1:1775772205.230775:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120ca200. 00010000:00000010:0.1:1775772205.230784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88e00. 00010000:00000010:2.1:1775772205.659188:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a500. 00010000:00000010:2.1:1775772205.659203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27c00. 00010000:00000010:2.1:1775772205.659207:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bed800. 00010000:00000010:2.1:1775772205.659211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226004200. 00000080:00000010:2.1:1775772205.670267:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f8e0. 00000080:00000010:2.1:1775772205.670293:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f023180. 00010000:00000010:2.1:1775772205.679515:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a500. 00010000:00000010:2.1:1775772205.679527:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27c00. 00010000:00000010:2.1:1775772205.679531:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6700. 00010000:00000010:2.1:1775772205.679534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00000080:00000010:2.1:1775772205.689292:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fb60. 00000080:00000010:2.1:1775772205.694305:0:206798:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185cb180. 00010000:00000010:2.1:1775772205.700188:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a500. 00010000:00000010:2.1:1775772205.700204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27c00. 00010000:00000010:2.1:1775772205.700208:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64c00. 00010000:00000010:2.1:1775772205.700212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101152a00. 00000080:00000010:2.1:1775772205.706349:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f8e0. 00000080:00000010:2.1:1775772205.706375:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f0239c0. 00010000:00000010:2.1:1775772205.716182:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a500. 00010000:00000010:2.1:1775772205.716197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27c00. 00010000:00000010:2.1:1775772205.716202:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64200. 00010000:00000010:2.1:1775772205.716207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44400. 00000080:00000010:2.1:1775772205.728384:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fb60. 00000080:00000010:2.1:1775772205.728430:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192c98c0. 00010000:00000010:2.1:1775772205.744255:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a500. 00010000:00000010:2.1:1775772205.762258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc48c00. 00010000:00000010:2.1:1775772205.762273:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120ca500. 00010000:00000010:2.1:1775772205.762278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89800. 00000080:00000010:2.1:1775772205.772163:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd4e0. 00000080:00000010:2.1:1775772205.772182:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f025280. 00010000:00000010:2.1:1775772205.788258:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a500. 00010000:00000010:2.1:1775772205.788269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27400. 00010000:00000010:2.1:1775772205.788274:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bed900. 00010000:00000010:2.1:1775772205.788279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226005a00. 00000080:00000010:2.1:1775772205.800737:0:206798:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fb60. 00000080:00000010:2.1:1775772205.817244:0:206798:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226bc4200. 00010000:00000010:2.1:1775772205.827250:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a500. 00010000:00000010:2.1:1775772205.827258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27400. 00010000:00000010:2.1:1775772205.827261:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6100. 00010000:00000010:2.1:1775772205.827264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:0.1:1775772205.830930:0:206798:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26a00. 00010000:00000010:0.1:1775772205.830940:0:206798:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64300. 00010000:00000010:0.1:1775772205.830945:0:206798:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7c00. 00000080:00000010:2.1:1775772205.831513:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f040. 00000080:00000010:2.1:1775772205.831533:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226bc6300. 00010000:00000010:2.1:1775772205.838179:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a100. 00000080:00000010:0.1:1775772205.838456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fb60. 00000080:00000010:0.1:1775772205.841510:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192cc200. 00010000:00000010:2.1:1775772205.853015:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120ca200. 00010000:00000010:2.1:1775772205.860312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89e00. 00010000:00000010:2.1:1775772205.860321:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64a00. 00010000:00000010:2.1:1775772205.860325:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44600. 00000080:00000010:2.1:1775772205.868366:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f160. 00000080:00000010:2.1:1775772205.868415:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192ce300. 00010000:00000010:2.1:1775772205.876365:0:206798:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a100. 00010000:00000010:2.1:1775772205.876371:0:206798:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27400. 00010000:00000010:2.1:1775772205.876374:0:206798:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120caa00. 00010000:00000010:2.1:1775772205.876378:0:206798:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89400. 00000080:00000010:2.1:1775772205.888708:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f040. 00000080:00000010:2.1:1775772205.888738:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192cd280. 00010000:00000010:0.1:1775772205.899317:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26000. 00010000:00000010:0.1:1775772205.899334:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64b00. 00010000:00000010:0.1:1775772205.899338:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7800. 00010000:00000010:2.1:1775772205.907264:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a500. 00000080:00000010:0.1:1775772205.918599:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f100. 00000080:00000010:0.1:1775772205.918627:0:13:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192cb9c0. 00010000:00000010:2.1:1775772205.922738:0:206798:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120caa00. 00010000:00000010:2.1:1775772205.922748:0:206798:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89a00. 00010000:00000010:2.1:1775772205.922752:0:206798:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621022f700. 00010000:00000010:2.1:1775772205.922756:0:206798:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00000080:00000010:2.1:1775772205.930649:0:206798:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fc20. 00000080:00000010:2.1:1775772205.943534:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185cca40. 00010000:00000010:2.1:1775772205.947217:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a500. 00010000:00000010:2.1:1775772205.947224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27400. 00010000:00000010:2.1:1775772205.947227:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06ad00. 00010000:00000010:2.1:1775772205.947230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00000080:00000010:2.1:1775772205.950297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f040. 00000080:00000010:2.1:1775772205.950322:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185ce300. 00010000:00000010:2.1:1775772205.959196:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a100. 00010000:00000010:2.1:1775772205.959208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26600. 00010000:00000010:2.1:1775772205.959211:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bedb00. 00010000:00000010:2.1:1775772205.959213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226004e00. 00000080:00000010:2.1:1775772205.968318:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fc40. 00000080:00000010:2.1:1775772205.968350:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226bc18c0. 00010000:00000010:2.1:1775772205.976336:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06ad00. 00010000:00000010:2.1:1775772205.976349:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27400. 00010000:00000010:2.1:1775772205.976352:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a000. 00010000:00000010:2.1:1775772205.976354:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27600. 00000080:00000010:2.1:1775772205.982323:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f040. 00000080:00000010:2.1:1775772205.982346:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185c98c0. 00010000:00000010:2.1:1775772205.992800:0:206798:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a100. 00010000:00000010:2.1:1775772206.001230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26600. 00010000:00000010:2.1:1775772206.001236:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120ca300. 00010000:00000010:2.1:1775772206.001240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88400. 00000080:00000010:2.1:1775772206.013906:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fc40. 00000080:00000010:2.1:1775772206.018368:0:206462:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192db180. 00010000:00000010:2.1:1775772206.018376:0:206462:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a100. 00000080:00000010:0.1:1775772206.026420:0:206463:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f040. 00000080:00000010:0.1:1775772206.026444:0:206463:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192c9080. 00010000:00000010:2.1:1775772206.760674:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226005600. 00010000:00000010:2.1:1775772206.766833:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bed600. 00010000:00000010:2.1:1775772206.766848:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226005c00. 00000080:00000010:2.1:1775772212.723468:0:206848:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd160. 00010000:00000010:2.1:1775772212.753279:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06ad00. 00010000:00000010:0.1:1775772212.757449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26c00. 00010000:00000010:0.1:1775772218.071359:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bede00. 00010000:00000010:0.1:1775772218.071383:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226005a00. 00010000:00000010:2.1:1775772218.075183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89a00. 00010000:00000010:2.1:1775772218.196282:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120cab00. 00010000:00000010:2.1:1775772231.624253:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06ad00. 00010000:00000010:2.1:1775772231.624261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27600. 00010000:00000010:2.1:1775772231.630287:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64d00. 00010000:00000010:2.1:1775772231.630304:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc48400. 00010000:00000010:2.1:1775772231.640968:0:207550:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a900. 00010000:00000010:2.1:1775772231.791210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88600. 00010000:00000010:2.1:1775772231.791231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88800. 00010000:00000010:2.1:1775772231.791237:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06ad00. 00010000:00000010:2.1:1775772231.791243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27600. 00010000:00000010:2.1:1775772231.791248:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bed100. 00010000:00000010:2.1:1775772231.791251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226004c00. 00000080:00000010:2.1:1775772231.806413:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f480. 00000080:00000010:2.1:1775772231.806448:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192ceb40. 00010000:00000010:2.1:1775772231.933989:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64300. 00010000:00000010:2.1:1775772231.934005:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc48200. 00010000:00000010:2.1:1775772231.938199:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06ad00. 00010000:00000010:2.1:1775772231.938221:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06af00. 00010000:00000010:2.1:1775772231.938225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27600. 00010000:00000010:2.1:1775772232.029181:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64b00. 00010000:00000010:2.1:1775772232.029192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc48200. 00010000:00000010:2.1:1775772232.057675:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06ad00. 00010000:00000010:2.1:1775772232.060176:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a000. 00010000:00000010:2.1:1775772232.060183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27800. 00000080:00000010:2.1:1775772232.101410:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e240. 00010000:00000010:2.1:1775772232.107333:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a000. 00010000:00000010:2.1:1775772232.107352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27800. 00010000:00000010:2.1:1775772232.131576:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a900. 00010000:00000010:2.1:1775772232.131596:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a000. 00010000:00000010:2.1:1775772232.131601:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26800. 00000080:00000010:2.1:1775772232.225566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e240. 00010000:00000010:2.1:1775772232.237523:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a000. 00010000:00000010:2.1:1775772232.237543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26800. 00010000:00000010:2.1:1775772232.249187:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a900. 00010000:00000010:2.1:1775772232.249201:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a000. 00010000:00000010:2.1:1775772232.249205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27800. 00000080:00000010:2.1:1775772232.285347:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e240. 00010000:00000010:2.1:1775772232.304207:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a900. 00010000:00000010:2.1:1775772232.304224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26800. 00010000:00000010:2.1:1775772232.310788:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a100. 00010000:00000010:2.1:1775772232.310805:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06ae00. 00010000:00000010:2.1:1775772232.310809:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26400. 00000080:00000010:2.1:1775772232.387292:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdac0. 00010000:00000010:2.1:1775772232.397253:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06ae00. 00010000:00000010:2.1:1775772232.397271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26400. 00010000:00000010:2.1:1775772232.406022:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a100. 00010000:00000010:2.1:1775772232.406028:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06ae00. 00010000:00000010:2.1:1775772232.406033:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27200. 00000080:00000010:2.1:1775772232.434298:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e280. 00010000:00000010:2.1:1775772232.439181:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a100. 00010000:00000010:2.1:1775772232.439190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26400. 00010000:00000010:2.1:1775772232.444681:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a600. 00010000:00000010:2.1:1775772232.448173:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a500. 00010000:00000010:2.1:1775772232.448181:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27e00. 00000080:00000010:2.1:1775772232.469345:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0edc0. 00010000:00000010:2.1:1775772232.475210:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a500. 00010000:00000010:2.1:1775772232.475225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27e00. 00010000:00000010:2.1:1775772232.483185:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a600. 00010000:00000010:2.1:1775772232.483199:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a500. 00010000:00000010:2.1:1775772232.483203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27a00. 00000080:00000010:2.1:1775772232.510181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e000. 00010000:00000010:2.1:1775772232.526331:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a600. 00010000:00000010:2.1:1775772232.526346:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27e00. 00010000:00000010:2.1:1775772232.531016:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06ac00. 00010000:00000010:2.1:1775772232.531029:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a800. 00010000:00000010:2.1:1775772232.531033:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27000. 00000080:00000010:0.1:1775772232.557305:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e000. 00010000:00000010:0.1:1775772232.561643:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120ca500. 00010000:00000010:0.1:1775772232.561650:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88000. 00010000:00000010:0.1:1775772232.568308:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120caf00. 00010000:00000010:0.1:1775772232.581170:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120caa00. 00010000:00000010:0.1:1775772232.581190:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88600. 00000080:00000010:0.1:1775772232.603412:0:2404:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fa20. 00010000:00000010:2.1:1775772232.613515:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120caa00. 00010000:00000010:2.1:1775772232.613524:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89000. 00010000:00000010:2.1:1775772232.618421:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a800. 00010000:00000010:2.1:1775772232.618431:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06ac00. 00010000:00000010:2.1:1775772232.618434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27000. 00000080:00000010:2.1:1775772232.645189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e1c0. 00010000:00000010:2.1:1775772232.652306:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06ac00. 00010000:00000010:2.1:1775772232.652319:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27000. 00010000:00000010:2.1:1775772232.659588:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a800. 00010000:00000010:2.1:1775772232.668603:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06ac00. 00010000:00000010:2.1:1775772232.668612:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26e00. 00000080:00000010:2.1:1775772232.682174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e1c0. 00010000:00000010:2.1:1775772232.691550:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06ac00. 00010000:00000010:2.1:1775772232.691565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26e00. 00010000:00000010:2.1:1775772232.705323:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a300. 00010000:00000010:2.1:1775772232.705335:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06ac00. 00010000:00000010:2.1:1775772232.705339:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26600. 00000080:00000010:2.1:1775772232.740168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e1c0. 00010000:00000010:2.1:1775772232.758200:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a300. 00010000:00000010:2.1:1775772232.758217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26e00. 00010000:00000010:2.1:1775772232.771350:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621022f700. 00010000:00000010:0.1:1775772232.890384:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64d00. 00010000:00000010:0.1:1775772232.890398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc49200. 00000080:00000010:2.1:1775772232.919255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd000. 00010000:00000010:2.1:1775772232.927250:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621022f700. 00010000:00000010:2.1:1775772232.927258:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26c00. 00010000:00000010:2.1:1775772232.944325:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6100. 00010000:00000010:2.1:1775772232.944338:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6700. 00010000:00000010:2.1:1775772232.944342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26a00. 00000080:00000010:2.1:1775772232.962889:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e1e0. 00010000:00000010:2.1:1775772232.972193:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6700. 00010000:00000010:2.1:1775772232.972206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26a00. 00010000:00000010:2.1:1775772232.985164:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6100. 00010000:00000010:2.1:1775772232.985177:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6700. 00010000:00000010:2.1:1775772232.985181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26000. 00000080:00000010:2.1:1775772233.016451:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e1e0. 00010000:00000010:2.1:1775772233.016474:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6700. 00010000:00000010:2.1:1775772233.016479:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26000. 00010000:00000010:2.1:1775772233.093513:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64300. 00010000:00000010:2.1:1775772233.093530:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc48000. 00010000:00000010:2.1:1775772233.098175:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6700. 00010000:00000010:2.1:1775772233.098183:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6100. 00010000:00000010:2.1:1775772233.098186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44600. 00000080:00000010:2.1:1775772233.118755:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e8c0. 00010000:00000010:2.1:1775772233.122328:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8b500. 00010000:00000010:2.1:1775772233.122344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44600. 00010000:00000010:2.1:1775772233.134174:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8b400. 00010000:00000010:2.1:1775772233.141381:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8b500. 00010000:00000010:2.1:1775772233.141389:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26000. 00000080:00000010:2.1:1775772233.168328:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e820. 00010000:00000010:2.1:1775772233.172777:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8b500. 00010000:00000010:2.1:1775772233.172786:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26000. 00010000:00000010:2.1:1775772233.176526:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8b200. 00010000:00000010:2.1:1775772233.192187:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8b000. 00010000:00000010:2.1:1775772233.192193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101152a00. 00010000:00000010:2.1:1775772233.249176:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64d00. 00010000:00000010:2.1:1775772233.249191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc49800. 00010000:00000010:2.1:1775772233.257489:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8b000. 00010000:00000010:2.1:1775772233.257501:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8b500. 00010000:00000010:2.1:1775772233.257505:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101152a00. 00000080:00000010:2.1:1775772233.286636:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e360. 00010000:00000010:2.1:1775772233.290307:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8b000. 00010000:00000010:2.1:1775772233.290318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:2.1:1775772233.300350:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8b900. 00010000:00000010:2.1:1775772233.305249:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8b000. 00010000:00000010:2.1:1775772233.305257:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00000080:00000010:2.1:1775772233.339170:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e080. 00010000:00000010:2.1:1775772233.346883:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8b000. 00010000:00000010:2.1:1775772233.346894:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:2.1:1775772233.351250:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8b300. 00010000:00000010:2.1:1775772233.351258:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8bc00. 00010000:00000010:2.1:1775772233.351262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00000080:00000010:2.1:1775772233.383175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e120. 00010000:00000010:0.1:1775772233.404265:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8bc00. 00010000:00000010:0.1:1775772233.404279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:0.1:1775772233.415428:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120ca200. 00010000:00000010:0.1:1775772233.415441:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120cad00. 00010000:00000010:0.1:1775772233.415445:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89200. 00000080:00000010:2.1:1775772233.434160:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e560. 00010000:00000010:2.1:1775772233.442245:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8b300. 00010000:00000010:2.1:1775772233.442253:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00010000:00000010:2.1:1775772233.456176:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8bb00. 00010000:00000010:2.1:1775772233.456189:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8b800. 00010000:00000010:2.1:1775772233.456194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84400. 00000080:00000010:2.1:1775772233.467276:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e120. 00010000:00000010:2.1:1775772233.477725:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8b800. 00010000:00000010:2.1:1775772233.477732:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84400. 00010000:00000010:2.1:1775772233.484216:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8bb00. 00010000:00000010:2.1:1775772233.484224:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8bc00. 00010000:00000010:2.1:1775772233.484228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85200. 00000080:00000010:2.1:1775772233.503168:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0ec00. 00010000:00000010:2.1:1775772233.521308:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8bc00. 00010000:00000010:2.1:1775772233.521322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85200. 00010000:00000010:0.1:1775772233.525194:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8bb00. 00010000:00000010:0.1:1775772233.525204:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120cad00. 00010000:00000010:0.1:1775772233.525208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00000080:00000010:2.1:1775772233.555245:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0ec00. 00010000:00000010:2.1:1775772233.555276:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8bc00. 00010000:00000010:2.1:1775772233.555282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85200. 00010000:00000010:2.1:1775772233.570358:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8bb00. 00010000:00000010:2.1:1775772233.570374:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8bc00. 00010000:00000010:2.1:1775772233.570377:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00000080:00000010:2.1:1775772233.655206:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e460. 00010000:00000010:2.1:1775772233.669460:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8bb00. 00010000:00000010:2.1:1775772233.669476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85200. 00000080:00000010:2.1:1775772233.750294:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0eb40. 00010000:00000010:2.1:1775772233.763316:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6100. 00010000:00000010:2.1:1775772233.763333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d9e00. 00010000:00000010:2.1:1775772233.783308:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8bb00. 00010000:00000010:2.1:1775772233.783323:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6100. 00010000:00000010:2.1:1775772233.783328:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d9000. 00000080:00000010:2.1:1775772233.868274:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0eb40. 00010000:00000010:2.1:1775772233.907519:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8bb00. 00010000:00000010:2.1:1775772233.907531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d9e00. 00010000:00000010:2.1:1775772233.910379:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453d00. 00010000:00000010:2.1:1775772233.914420:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453e00. 00010000:00000010:2.1:1775772233.914427:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d9a00. 00010000:00000010:2.1:1775772234.030299:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111000. 00010000:00000010:2.1:1775772234.030307:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc49400. 00000080:00000010:2.1:1775772234.091312:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd9c0. 00010000:00000010:2.1:1775772234.116545:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453d00. 00010000:00000010:2.1:1775772234.116558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d9e00. 00010000:00000010:2.1:1775772234.121253:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453b00. 00010000:00000010:2.1:1775772234.121262:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453c00. 00010000:00000010:2.1:1775772234.121266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8200. 00000080:00000010:2.1:1775772234.150193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e2e0. 00010000:00000010:2.1:1775772234.155187:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453c00. 00010000:00000010:2.1:1775772234.155202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8200. 00010000:00000010:2.1:1775772234.167187:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453b00. 00010000:00000010:2.1:1775772234.176294:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453c00. 00010000:00000010:2.1:1775772234.176305:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8e00. 00000080:00000010:2.1:1775772234.195341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e3c0. 00010000:00000010:2.1:1775772234.201226:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453c00. 00010000:00000010:2.1:1775772234.201240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8e00. 00010000:00000010:2.1:1775772234.210823:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453900. 00010000:00000010:2.1:1775772234.210834:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453c00. 00010000:00000010:2.1:1775772234.210838:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d9600. 00000080:00000010:2.1:1775772234.238339:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0ea20. 00010000:00000010:2.1:1775772234.248849:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453900. 00010000:00000010:2.1:1775772234.248857:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8e00. 00010000:00000010:2.1:1775772234.255181:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453700. 00010000:00000010:2.1:1775772234.255195:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453600. 00010000:00000010:2.1:1775772234.255199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8c00. 00000080:00000010:2.1:1775772234.349211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e880. 00010000:00000010:2.1:1775772234.359185:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453700. 00010000:00000010:2.1:1775772234.359199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8e00. 00010000:00000010:2.1:1775772234.375235:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453900. 00010000:00000010:2.1:1775772234.413195:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111b00. 00010000:00000010:2.1:1775772234.413210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc49600. 00010000:00000010:2.1:1775772234.427517:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453900. 00010000:00000010:2.1:1775772234.427529:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453000. 00010000:00000010:2.1:1775772234.427533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8e00. 00000080:00000010:2.1:1775772234.455652:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fda00. 00000080:00000010:2.1:1775772234.507903:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792cc40. 00010000:00000010:2.1:1775772234.513189:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453a00. 00010000:00000010:2.1:1775772234.513204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8a00. 00010000:00000010:2.1:1775772234.521921:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453800. 00010000:00000010:2.1:1775772234.530222:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453a00. 00010000:00000010:2.1:1775772234.530230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8000. 00000080:00000010:2.1:1775772234.548637:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0eba0. 00000080:00000010:2.1:1775772234.588184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792ce20. 00010000:00000010:2.1:1775772234.605271:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453800. 00010000:00000010:2.1:1775772234.605280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8a00. 00010000:00000010:2.1:1775772234.609350:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453300. 00010000:00000010:2.1:1775772234.609363:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453f00. 00010000:00000010:2.1:1775772234.609368:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8600. 00000080:00000010:2.1:1775772234.638280:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0eba0. 00010000:00000010:2.1:1775772234.648193:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453300. 00010000:00000010:2.1:1775772234.648210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8a00. 00010000:00000010:2.1:1775772234.657313:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453800. 00010000:00000010:2.1:1775772234.661273:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453300. 00010000:00000010:2.1:1775772234.661283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d9800. 00000080:00000010:2.1:1775772234.696276:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0eba0. 00010000:00000010:2.1:1775772234.701233:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453300. 00010000:00000010:2.1:1775772234.701249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d9800. 00010000:00000010:2.1:1775772234.708982:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453200. 00010000:00000010:2.1:1775772234.708989:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453300. 00010000:00000010:2.1:1775772234.708993:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8800. 00000080:00000010:2.1:1775772234.751170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0eb40. 00000080:00000010:0.1:1775772234.782386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd3a0. 00010000:00000010:0.1:1775772234.793516:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120cad00. 00010000:00000010:0.1:1775772234.793524:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89200. 00010000:00000010:0.1:1775772234.799270:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120cab00. 00010000:00000010:0.1:1775772234.804918:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120cad00. 00010000:00000010:0.1:1775772234.804926:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88200. 00000080:00000010:0.1:1775772234.827169:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fae0. 00010000:00000010:0.1:1775772234.844618:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120cad00. 00010000:00000010:0.1:1775772234.844625:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89800. 00010000:00000010:0.1:1775772234.856221:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120ca000. 00010000:00000010:2.1:1775772234.875179:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120caa00. 00010000:00000010:2.1:1775772234.875188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89a00. 00000080:00000010:2.1:1775772234.894266:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fae0. 00010000:00000010:2.1:1775772234.911832:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453200. 00010000:00000010:2.1:1775772234.911845:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8000. 00010000:00000010:2.1:1775772234.921292:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453a00. 00010000:00000010:2.1:1775772234.921304:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453900. 00010000:00000010:2.1:1775772234.921308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d9400. 00010000:00000010:2.1:1775772235.016183:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111400. 00010000:00000010:2.1:1775772235.016194:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453900. 00010000:00000010:2.1:1775772235.016199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc49600. 00000080:00000010:2.1:1775772235.035465:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd0a0. 00010000:00000010:2.1:1775772235.066329:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111900. 00010000:00000010:2.1:1775772235.066342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc49400. 00000080:00000010:2.1:1775772235.094844:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fda20. 00010000:00000010:2.1:1775772235.098529:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453700. 00010000:00000010:2.1:1775772235.098544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8400. 00010000:00000010:2.1:1775772235.118599:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453900. 00010000:00000010:2.1:1775772235.118610:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453700. 00010000:00000010:2.1:1775772235.118613:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d9400. 00000080:00000010:2.1:1775772235.187928:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e1e0. 00010000:00000010:2.1:1775772235.212193:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453900. 00010000:00000010:2.1:1775772235.212210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8400. 00010000:00000010:2.1:1775772235.239552:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8bb00. 00010000:00000010:2.1:1775772235.239569:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453900. 00010000:00000010:2.1:1775772235.239574:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85200. 00000080:00000010:2.1:1775772235.285204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e1e0. 00010000:00000010:2.1:1775772235.305301:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8bb00. 00010000:00000010:2.1:1775772235.305309:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8400. 00010000:00000010:2.1:1775772235.329228:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414d00. 00010000:00000010:2.1:1775772235.329241:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414000. 00010000:00000010:2.1:1775772235.329244:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ad200. 00000080:00000010:2.1:1775772235.344491:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e1e0. 00010000:00000010:2.1:1775772235.351274:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414000. 00010000:00000010:2.1:1775772235.351288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ad200. 00010000:00000010:2.1:1775772235.367321:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414d00. 00010000:00000010:2.1:1775772235.367328:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414000. 00010000:00000010:2.1:1775772235.367331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ade00. 00000080:00000010:2.1:1775772235.406540:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0ebe0. 00010000:00000010:2.1:1775772235.419658:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414000. 00010000:00000010:2.1:1775772235.419673:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ade00. 00010000:00000010:2.1:1775772235.435233:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414d00. 00010000:00000010:0.1:1775772235.468801:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111100. 00010000:00000010:0.1:1775772235.468815:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc49200. 00010000:00000010:0.1:1775772235.489580:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120ca000. 00010000:00000010:0.1:1775772235.489593:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120cad00. 00010000:00000010:0.1:1775772235.489597:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89a00. 00000080:00000010:0.1:1775772235.514316:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdaa0. 00010000:00000010:0.1:1775772235.521933:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120cad00. 00010000:00000010:0.1:1775772235.521949:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89a00. 00010000:00000010:0.1:1775772235.532193:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120ca000. 00010000:00000010:0.1:1775772235.541221:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120cad00. 00010000:00000010:0.1:1775772235.541239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89800. 00000080:00000010:2.1:1775772235.557668:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fe60. 00010000:00000010:2.1:1775772235.563178:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414300. 00010000:00000010:2.1:1775772235.563192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ada00. 00010000:00000010:2.1:1775772235.576474:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414a00. 00010000:00000010:2.1:1775772235.576487:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414300. 00010000:00000010:2.1:1775772235.576490:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ac200. 00000080:00000010:2.1:1775772235.604188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0ebe0. 00010000:00000010:2.1:1775772235.613272:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414a00. 00010000:00000010:2.1:1775772235.613287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ada00. 00010000:00000010:2.1:1775772235.622216:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414800. 00010000:00000010:2.1:1775772235.622222:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414500. 00010000:00000010:2.1:1775772235.622226:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ace00. 00000080:00000010:2.1:1775772235.638188:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e7e0. 00010000:00000010:2.1:1775772235.648630:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414500. 00010000:00000010:2.1:1775772235.648637:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ace00. 00010000:00000010:2.1:1775772235.659169:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414800. 00010000:00000010:2.1:1775772235.659181:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414f00. 00010000:00000010:2.1:1775772235.659185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ad600. 00000080:00000010:2.1:1775772235.678475:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e7e0. 00010000:00000010:2.1:1775772235.691248:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414f00. 00010000:00000010:2.1:1775772235.691256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ad600. 00010000:00000010:2.1:1775772235.691259:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414800. 00010000:00000010:2.1:1775772235.703714:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414200. 00010000:00000010:2.1:1775772235.703721:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262acc00. 00000080:00000010:2.1:1775772235.776230:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdd80. 00000080:00000010:2.1:1775772235.817599:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd560. 00010000:00000010:2.1:1775772235.817615:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414900. 00010000:00000010:2.1:1775772235.817619:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ad400. 00010000:00000010:2.1:1775772235.822259:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414600. 00010000:00000010:2.1:1775772235.822271:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414b00. 00010000:00000010:2.1:1775772235.822276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262aca00. 00000080:00000010:2.1:1775772235.855314:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0eb40. 00010000:00000010:2.1:1775772235.866309:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414600. 00010000:00000010:2.1:1775772235.866324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ad400. 00010000:00000010:2.1:1775772235.875171:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414900. 00010000:00000010:2.1:1775772235.878349:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414600. 00010000:00000010:2.1:1775772235.878361:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ac000. 00000080:00000010:2.1:1775772235.908715:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0eb40. 00010000:00000010:2.1:1775772235.913194:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414600. 00010000:00000010:2.1:1775772235.913209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ac000. 00010000:00000010:2.1:1775772235.920185:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414100. 00010000:00000010:2.1:1775772235.925230:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414600. 00010000:00000010:2.1:1775772235.925239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ac600. 00000080:00000010:2.1:1775772235.944360:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b1689e0. 00010000:00000010:2.1:1775772235.950328:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414600. 00010000:00000010:2.1:1775772235.950334:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ac600. 00010000:00000010:2.1:1775772235.954304:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414400. 00010000:00000010:2.1:1775772235.959180:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414c00. 00010000:00000010:2.1:1775772235.959189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ad800. 00000080:00000010:2.1:1775772235.976206:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6220367d00. 00010000:00000010:2.1:1775772235.981431:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414c00. 00010000:00000010:2.1:1775772235.981440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ad800. 00010000:00000010:2.1:1775772235.981443:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414600. 00010000:00000010:2.1:1775772235.985185:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414f00. 00010000:00000010:2.1:1775772235.985195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ac800. 00000080:00000010:2.1:1775772235.995179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203674e0. 00010000:00000010:2.1:1775772236.000272:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414f00. 00010000:00000010:2.1:1775772236.000278:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ac800. 00010000:00000010:2.1:1775772236.009234:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414600. 00010000:00000010:2.1:1775772236.009239:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414c00. 00010000:00000010:2.1:1775772236.009241:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ad800. 00000080:00000010:2.1:1775772236.028182:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6220367d00. 00010000:00000010:2.1:1775772236.033226:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414c00. 00010000:00000010:2.1:1775772236.033242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ad800. 00010000:00000010:2.1:1775772236.056179:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414600. 00010000:00000010:2.1:1775772236.056194:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8bb00. 00010000:00000010:2.1:1775772236.056199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262adc00. 00000080:00000010:2.1:1775772236.079997:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa420. 00010000:00000010:2.1:1775772236.085258:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414600. 00010000:00000010:2.1:1775772236.085275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262adc00. 00010000:00000010:2.1:1775772236.096262:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8bb00. 00010000:00000010:2.1:1775772236.101287:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414600. 00010000:00000010:2.1:1775772236.101302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ad000. 00000080:00000010:2.1:1775772236.137877:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa680. 00010000:00000010:2.1:1775772236.141589:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414600. 00010000:00000010:2.1:1775772236.141596:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ad000. 00010000:00000010:2.1:1775772236.150547:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205500. 00010000:00000010:2.1:1775772236.153348:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205c00. 00010000:00000010:2.1:1775772236.153356:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8400. 00000080:00000010:2.1:1775772236.169311:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00010000:00000010:2.1:1775772236.174192:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205c00. 00010000:00000010:2.1:1775772236.174196:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8400. 00010000:00000010:2.1:1775772236.182256:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205500. 00010000:00000010:2.1:1775772236.182266:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205c00. 00010000:00000010:2.1:1775772236.182270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bea00. 00000080:00000010:2.1:1775772236.198337:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa120. 00010000:00000010:2.1:1775772236.203191:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205c00. 00010000:00000010:2.1:1775772236.203208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bea00. 00010000:00000010:2.1:1775772236.214204:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205500. 00010000:00000010:2.1:1775772236.214209:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205c00. 00010000:00000010:2.1:1775772236.214212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348be600. 00000080:00000010:2.1:1775772236.235291:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6220367d00. 00010000:00000010:2.1:1775772236.251418:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205c00. 00010000:00000010:2.1:1775772236.251434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348be600. 00010000:00000010:2.1:1775772236.270186:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205500. 00010000:00000010:0.1:1775772236.270240:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205c00. 00010000:00000010:0.1:1775772236.270246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bf800. 00000080:00000010:2.1:1775772236.330102:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792cd20. 00010000:00000010:2.1:1775772236.337205:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205500. 00010000:00000010:2.1:1775772236.337212:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348be600. 00010000:00000010:2.1:1775772236.344186:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205100. 00010000:00000010:2.1:1775772236.344197:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205500. 00010000:00000010:2.1:1775772236.344201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348be800. 00000080:00000010:2.1:1775772236.365172:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6220367d00. 00010000:00000010:2.1:1775772236.374186:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205100. 00010000:00000010:2.1:1775772236.374201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348be600. 00010000:00000010:2.1:1775772236.381416:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205400. 00010000:00000010:2.1:1775772236.385288:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205100. 00010000:00000010:2.1:1775772236.385301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348be400. 00000080:00000010:2.1:1775772236.416881:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620130bbe0. 00010000:00000010:2.1:1775772236.424240:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205400. 00010000:00000010:2.1:1775772236.424248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348be600. 00010000:00000010:2.1:1775772236.438730:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205300. 00010000:00000010:2.1:1775772236.438743:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205d00. 00010000:00000010:2.1:1775772236.438748:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bf200. 00000080:00000010:2.1:1775772236.456166:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620130be60. 00010000:00000010:2.1:1775772236.464223:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205d00. 00010000:00000010:2.1:1775772236.464240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bf200. 00010000:00000010:2.1:1775772236.476211:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205300. 00010000:00000010:2.1:1775772236.479324:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205d00. 00010000:00000010:2.1:1775772236.479331:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bfe00. 00000080:00000010:2.1:1775772236.497345:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620130be60. 00010000:00000010:2.1:1775772236.506428:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205d00. 00010000:00000010:2.1:1775772236.506439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bfe00. 00010000:00000010:2.1:1775772236.513174:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205300. 00010000:00000010:2.1:1775772236.516349:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205d00. 00010000:00000010:2.1:1775772236.516355:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bf000. 00000080:00000010:2.1:1775772236.557073:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620130b180. 00010000:00000010:2.1:1775772236.557094:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205d00. 00010000:00000010:2.1:1775772236.557099:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bf000. 00010000:00000010:2.1:1775772236.561194:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205900. 00010000:00000010:2.1:1775772236.561207:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205200. 00010000:00000010:2.1:1775772236.561211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bfa00. 00010000:00000010:0.1:1775772236.653179:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111200. 00010000:00000010:0.1:1775772236.653188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc49e00. 00010000:00000010:2.1:1775772236.661439:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120ca000. 00010000:00000010:2.1:1775772236.661446:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205900. 00010000:00000010:2.1:1775772236.661450:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89a00. 00000080:00000010:2.1:1775772236.705223:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f820. 00010000:00000010:2.1:1775772236.710182:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205900. 00010000:00000010:2.1:1775772236.710193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348be200. 00000080:00000010:2.1:1775772236.722354:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620130b8e0. 00010000:00000010:2.1:1775772236.734571:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205900. 00010000:00000010:2.1:1775772236.734577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348be200. 00010000:00000010:2.1:1775772236.744907:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205800. 00010000:00000010:0.1:1775772236.750167:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205a00. 00010000:00000010:0.1:1775772236.750177:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bf600. 00000080:00000010:2.1:1775772236.772793:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620130bbe0. 00010000:00000010:0.1:1775772236.781579:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205800. 00010000:00000010:0.1:1775772236.781591:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bec00. 00010000:00000010:2.1:1775772236.813179:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120ca000. 00010000:00000010:2.1:1775772236.813188:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205a00. 00010000:00000010:2.1:1775772236.813192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88800. 00000080:00000010:2.1:1775772236.850720:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fa20. 00010000:00000010:2.1:1775772236.904091:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111300. 00010000:00000010:2.1:1775772236.904107:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc48200. 00010000:00000010:2.1:1775772236.907202:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205c00. 00010000:00000010:2.1:1775772236.911252:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205a00. 00010000:00000010:2.1:1775772236.911261:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bf400. 00000080:00000010:2.1:1775772236.923668:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620130bec0. 00010000:00000010:2.1:1775772236.932168:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205a00. 00010000:00000010:2.1:1775772236.932176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bf400. 00010000:00000010:2.1:1775772236.941167:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205c00. 00010000:00000010:2.1:1775772236.941177:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414600. 00010000:00000010:2.1:1775772236.941182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bf600. 00000080:00000010:2.1:1775772236.946815:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620130b180. 00010000:00000010:2.1:1775772236.952171:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205c00. 00010000:00000010:2.1:1775772236.952178:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bf600. 00010000:00000010:2.1:1775772236.964201:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414600. 00010000:00000010:2.1:1775772236.967427:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205c00. 00010000:00000010:2.1:1775772236.967434:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bee00. 00000080:00000010:0.1:1775772236.988319:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620130b780. 00010000:00000010:0.1:1775772236.988341:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120ca000. 00010000:00000010:0.1:1775772236.988347:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89000. 00010000:00000010:0.1:1775772237.010647:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5900. 00010000:00000010:0.1:1775772237.010670:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120ca000. 00010000:00000010:0.1:1775772237.010675:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88800. 00010000:00000010:2.1:1775772237.054174:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bedb00. 00010000:00000010:2.1:1775772237.054181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226005600. 00000080:00000010:2.1:1775772237.076310:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792c4e0. 00010000:00000010:2.1:1775772237.082214:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9900. 00010000:00000010:2.1:1775772237.082224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bf800. 00010000:00000010:2.1:1775772237.087212:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9e00. 00010000:00000010:2.1:1775772237.087218:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9800. 00010000:00000010:2.1:1775772237.087221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8400. 00000080:00000010:2.1:1775772237.115957:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620130bcc0. 00010000:00000010:2.1:1775772237.125173:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9800. 00010000:00000010:2.1:1775772237.125190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8400. 00010000:00000010:2.1:1775772237.133048:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9e00. 00010000:00000010:2.1:1775772237.137444:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9800. 00010000:00000010:2.1:1775772237.137458:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628b800. 00000080:00000010:0.1:1775772237.177861:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620130bcc0. 00010000:00000010:0.1:1775772237.183185:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0a00. 00010000:00000010:0.1:1775772237.183201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89a00. 00010000:00000010:0.1:1775772237.197265:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0700. 00010000:00000010:0.1:1775772237.205244:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0a00. 00010000:00000010:0.1:1775772237.205252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88800. 00000080:00000010:2.1:1775772237.240171:0:207602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f9a0. 00010000:00000010:2.1:1775772237.249598:0:207602:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9e00. 00010000:00000010:2.1:1775772237.249616:0:207602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628a800. 00010000:00000010:2.1:1775772237.255173:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9400. 00010000:00000010:2.1:1775772237.262434:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9e00. 00010000:00000010:2.1:1775772237.262441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628a400. 00000080:00000010:2.1:1775772237.284163:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00010000:00000010:2.1:1775772237.334422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628a800. 00010000:00000010:2.1:1775772237.334430:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120ca300. 00010000:00000010:2.1:1775772237.334434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89c00. 00010000:00000010:2.1:1775772237.334437:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9e00. 00010000:00000010:2.1:1775772237.334440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628a400. 00000080:00000010:2.1:1775772237.338322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:2.1:1775772237.338357:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192c8000. 00010000:00000010:0.1:1775772237.398231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628a400. 00010000:00000010:0.1:1775772237.398238:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06af00. 00010000:00000010:0.1:1775772237.398243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27600. 00010000:00000010:0.1:1775772237.398246:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9200. 00010000:00000010:0.1:1775772237.398249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628be00. 00000080:00000010:2.1:1775772237.405254:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:2.1:1775772237.405270:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185c98c0. 00010000:00000010:2.1:1775772237.422304:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628ba00. 00010000:00000010:2.1:1775772237.422312:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06ad00. 00010000:00000010:2.1:1775772237.422315:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27400. 00010000:00000010:2.1:1775772237.422317:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9c00. 00010000:00000010:2.1:1775772237.422320:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628b000. 00000080:00000010:2.1:1775772237.437059:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772237.437082:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185ce300. 00010000:00000010:2.1:1775772237.449296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628b000. 00010000:00000010:2.1:1775772237.449311:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64a00. 00010000:00000010:2.1:1775772237.449317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc48400. 00010000:00000010:2.1:1775772237.449321:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9000. 00010000:00000010:2.1:1775772237.449324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628ae00. 00000080:00000010:2.1:1775772237.468330:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:2.1:1775772237.468348:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185cca40. 00010000:00000010:2.1:1775772237.495322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628b000. 00010000:00000010:2.1:1775772237.495331:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a000. 00010000:00000010:2.1:1775772237.495335:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27800. 00010000:00000010:2.1:1775772237.495338:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9000. 00010000:00000010:2.1:1775772237.495341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628ae00. 00000080:00000010:2.1:1775772237.495345:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772237.495361:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185cb180. 00000080:00000010:0.1:1775772237.548835:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:0.1:1775772237.556168:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185cd280. 00010000:00000010:0.1:1775772237.570723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628ae00. 00010000:00000010:0.1:1775772237.570746:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64100. 00010000:00000010:0.1:1775772237.570752:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc49a00. 00010000:00000010:0.1:1775772237.570756:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89800. 00000080:00000010:2.1:1775772237.573260:0:13:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192cdac0. 00000080:00000010:0.1:1775772237.578301:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00010000:00000010:2.1:1775772237.635715:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27400. 00010000:00000010:2.1:1775772237.635729:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06ae00. 00010000:00000010:2.1:1775772237.635732:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27200. 00010000:00000010:2.1:1775772237.635735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628b400. 00010000:00000010:2.1:1775772237.727864:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27600. 00010000:00000010:2.1:1775772237.727881:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a100. 00010000:00000010:2.1:1775772237.727885:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26400. 00010000:00000010:2.1:1775772237.727888:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226005600. 00010000:00000010:2.1:1775772237.748211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27800. 00010000:00000010:2.1:1775772237.748223:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a500. 00010000:00000010:2.1:1775772237.748227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27a00. 00010000:00000010:2.1:1775772237.748229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101125800. 00000080:00000010:2.1:1775772237.770122:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdf40. 00000080:00000010:2.1:1775772237.770155:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185c8000. 00010000:00000010:0.1:1775772237.843246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7800. 00010000:00000010:0.1:1775772237.843262:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a600. 00010000:00000010:0.1:1775772237.843266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27e00. 00010000:00000010:0.1:1775772237.843269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628bc00. 00000080:00000010:2.1:1775772237.856856:0:207606:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772237.856877:0:207606:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185ceb40. 00010000:00000010:0.1:1775772237.891201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e103200. 00000080:00000010:0.1:1775772237.929470:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdc80. 00000080:00000010:0.1:1775772237.929494:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192dd280. 00000080:00000010:2.1:1775772237.999325:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792ca80. 00000080:00000010:2.1:1775772237.999354:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185c9080. 00010000:00000010:2.1:1775772238.025440:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296000. 00010000:00000010:2.1:1775772238.025448:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a800. 00010000:00000010:2.1:1775772238.025452:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27000. 00010000:00000010:2.1:1775772238.025455:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27400. 00000080:00000010:2.1:1775772238.063291:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772238.063309:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185cdac0. 00010000:00000010:2.1:1775772238.073961:0:207606:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296600. 00010000:00000010:2.1:1775772238.073969:0:207606:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06ac00. 00010000:00000010:2.1:1775772238.073983:0:207606:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26600. 00010000:00000010:2.1:1775772238.073989:0:207606:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26200. 00000080:00000010:2.1:1775772238.073994:0:207606:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:2.1:1775772238.074010:0:207606:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185cf380. 00010000:00000010:0.1:1775772238.099169:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc49600. 00010000:00000010:0.1:1775772238.099177:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64500. 00010000:00000010:0.1:1775772238.099182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26e00. 00010000:00000010:0.1:1775772238.099185:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111a00. 00010000:00000010:0.1:1775772238.099188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27400. 00000080:00000010:0.1:1775772238.139198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fa80. 00010000:00000010:2.1:1775772238.169066:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc49a00. 00010000:00000010:2.1:1775772238.169082:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120ca500. 00010000:00000010:2.1:1775772238.169086:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89600. 00010000:00000010:2.1:1775772238.169090:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3200. 00010000:00000010:2.1:1775772238.169092:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00000080:00000010:2.1:1775772238.173181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772238.173202:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226bc2100. 00010000:00000010:0.1:1775772238.183189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc48400. 00010000:00000010:0.1:1775772238.183207:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620b06a300. 00010000:00000010:0.1:1775772238.183212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27c00. 00010000:00000010:0.1:1775772238.183215:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3300. 00010000:00000010:0.1:1775772238.183218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8200. 00000080:00000010:0.1:1775772238.193287:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f480. 00000080:00000010:0.1:1775772238.197292:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192cf380. 00010000:00000010:0.1:1775772238.202166:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc48200. 00010000:00000010:0.1:1775772238.202171:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6e00. 00010000:00000010:0.1:1775772238.202176:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26c00. 00010000:00000010:0.1:1775772238.202178:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0500. 00010000:00000010:0.1:1775772238.202181:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297600. 00000080:00000010:0.1:1775772238.207257:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f480. 00000080:00000010:0.1:1775772238.207269:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185cb9c0. 00010000:00000010:0.1:1775772238.213391:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296800. 00010000:00000010:0.1:1775772238.213398:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621022f700. 00010000:00000010:0.1:1775772238.213402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26a00. 00010000:00000010:0.1:1775772238.213406:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0000. 00010000:00000010:0.1:1775772238.213423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296600. 00000080:00000010:0.1:1775772238.218790:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f060. 00000080:00000010:0.1:1775772238.218811:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185ca940. 00010000:00000010:0.1:1775772238.223198:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296400. 00010000:00000010:0.1:1775772238.223204:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64b00. 00010000:00000010:0.1:1775772238.223208:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc49000. 00010000:00000010:0.1:1775772238.223212:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0500. 00010000:00000010:0.1:1775772238.223215:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297600. 00000080:00000010:0.1:1775772238.232297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f520. 00000080:00000010:0.1:1775772238.232316:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192ceb40. 00010000:00000010:0.1:1775772238.239226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297200. 00010000:00000010:0.1:1775772238.239234:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8bd00. 00010000:00000010:0.1:1775772238.239238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44400. 00010000:00000010:0.1:1775772238.239241:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0000. 00010000:00000010:0.1:1775772238.239244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296600. 00000080:00000010:0.1:1775772238.239247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f060. 00000080:00000010:0.1:1775772238.239264:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f025280. 00010000:00000010:0.1:1775772238.244161:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297e00. 00010000:00000010:0.1:1775772238.244167:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8b400. 00010000:00000010:0.1:1775772238.244170:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44600. 00010000:00000010:0.1:1775772238.244173:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0500. 00010000:00000010:0.1:1775772238.244174:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297600. 00000080:00000010:0.1:1775772238.250165:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f520. 00000080:00000010:0.1:1775772238.250180:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f0239c0. 00010000:00000010:0.1:1775772238.257463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297000. 00010000:00000010:0.1:1775772238.257476:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8b200. 00010000:00000010:0.1:1775772238.257480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26000. 00010000:00000010:0.1:1775772238.257483:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0500. 00010000:00000010:0.1:1775772238.257485:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297600. 00000080:00000010:0.1:1775772238.265365:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f060. 00000080:00000010:0.1:1775772238.265393:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f023180. 00010000:00000010:2.1:1775772238.284396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620aa12c00. 00010000:00000010:2.1:1775772238.284412:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64300. 00010000:00000010:2.1:1775772238.284417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc48800. 00010000:00000010:2.1:1775772238.284420:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0500. 00010000:00000010:2.1:1775772238.284423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297600. 00000080:00000010:2.1:1775772238.306297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772238.309464:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192c9080. 00010000:00000010:2.1:1775772238.393157:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620aa12a00. 00010000:00000010:2.1:1775772238.393164:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8b000. 00010000:00000010:2.1:1775772238.393168:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:2.1:1775772238.393171:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990200. 00010000:00000010:2.1:1775772238.393174:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206b3d200. 00000080:00000010:2.1:1775772238.406259:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772238.406280:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62184a7380. 00000080:00000010:2.1:1775772238.436190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:2.1:1775772238.436208:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192db9c0. 00010000:00000010:2.1:1775772238.442185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620aa12600. 00010000:00000010:2.1:1775772238.442196:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8b300. 00010000:00000010:2.1:1775772238.442199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00010000:00000010:2.1:1775772238.442204:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3b00. 00010000:00000010:2.1:1775772238.442206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00000080:00000010:2.1:1775772238.490350:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd0e0. 00000080:00000010:2.1:1775772238.499273:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192d8840. 00010000:00000010:2.1:1775772238.504175:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620aa12400. 00010000:00000010:2.1:1775772238.504186:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8bc00. 00010000:00000010:2.1:1775772238.504191:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:2.1:1775772238.504195:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3b00. 00010000:00000010:2.1:1775772238.504198:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00000080:00000010:2.1:1775772238.519192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:2.1:1775772238.519220:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62184a5280. 00010000:00000010:2.1:1775772238.526317:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620aa13200. 00010000:00000010:2.1:1775772238.526327:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6700. 00010000:00000010:2.1:1775772238.526331:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d9200. 00010000:00000010:2.1:1775772238.526334:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3b00. 00010000:00000010:2.1:1775772238.526337:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00000080:00000010:2.1:1775772238.533196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772238.536375:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192cb9c0. 00010000:00000010:2.1:1775772238.546184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620aa13e00. 00010000:00000010:2.1:1775772238.546201:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6100. 00010000:00000010:2.1:1775772238.546206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d9000. 00010000:00000010:2.1:1775772238.546210:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3b00. 00010000:00000010:2.1:1775772238.546212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00000080:00000010:2.1:1775772238.556255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772238.560913:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62184a3180. 00010000:00000010:2.1:1775772238.573125:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620aa13000. 00010000:00000010:2.1:1775772238.573153:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64d00. 00010000:00000010:2.1:1775772238.573158:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc48600. 00010000:00000010:2.1:1775772238.573161:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3b00. 00010000:00000010:2.1:1775772238.573164:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00000080:00000010:2.1:1775772238.579108:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772238.579156:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62184a2940. 00010000:00000010:2.1:1775772238.594268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620aa13600. 00010000:00000010:2.1:1775772238.594277:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453e00. 00010000:00000010:2.1:1775772238.594281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d9a00. 00010000:00000010:2.1:1775772238.594285:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3500. 00010000:00000010:2.1:1775772238.594288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:2.1:1775772238.619982:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620aa12e00. 00010000:00000010:2.1:1775772238.619996:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453d00. 00010000:00000010:2.1:1775772238.620000:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d9e00. 00010000:00000010:2.1:1775772238.620003:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990600. 00010000:00000010:2.1:1775772238.620006:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206b3d000. 00000080:00000010:2.1:1775772238.635782:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fda60. 00000080:00000010:2.1:1775772238.635804:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62184a4a40. 00010000:00000010:2.1:1775772238.641182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620aa13a00. 00010000:00000010:2.1:1775772238.641201:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453b00. 00010000:00000010:2.1:1775772238.641205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8200. 00010000:00000010:2.1:1775772238.641208:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3500. 00010000:00000010:2.1:1775772238.641211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00000080:00000010:2.1:1775772238.650114:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:2.1:1775772238.650154:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62184a6300. 00010000:00000010:2.1:1775772238.654666:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211932c00. 00010000:00000010:2.1:1775772238.654675:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453c00. 00010000:00000010:2.1:1775772238.654679:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d9600. 00010000:00000010:2.1:1775772238.654682:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3b00. 00010000:00000010:2.1:1775772238.654685:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00000080:00000010:2.1:1775772238.663212:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772238.663230:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62184a18c0. 00010000:00000010:2.1:1775772238.679793:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211933c00. 00010000:00000010:2.1:1775772238.679810:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453600. 00010000:00000010:2.1:1775772238.679814:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8c00. 00010000:00000010:2.1:1775772238.679817:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3b00. 00010000:00000010:2.1:1775772238.679820:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00000080:00000010:2.1:1775772238.686821:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772238.686848:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62184a2100. 00010000:00000010:2.1:1775772238.720421:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211933800. 00010000:00000010:2.1:1775772238.720428:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453000. 00010000:00000010:2.1:1775772238.720431:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8e00. 00010000:00000010:2.1:1775772238.720434:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990e00. 00010000:00000010:2.1:1775772238.720437:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206b3de00. 00000080:00000010:2.1:1775772238.738217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd880. 00000080:00000010:2.1:1775772238.738240:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62184a4200. 00010000:00000010:2.1:1775772238.742180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211932800. 00010000:00000010:2.1:1775772238.742198:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bedf00. 00010000:00000010:2.1:1775772238.742204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226005c00. 00010000:00000010:2.1:1775772238.742207:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3b00. 00010000:00000010:2.1:1775772238.742210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00000080:00000010:2.1:1775772238.755679:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772238.762191:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226bc3180. 00010000:00000010:2.1:1775772238.781282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211932400. 00010000:00000010:2.1:1775772238.781298:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bed600. 00010000:00000010:2.1:1775772238.781303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226004e00. 00010000:00000010:2.1:1775772238.781306:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3b00. 00010000:00000010:2.1:1775772238.781309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772238.844501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211933e00. 00010000:00000010:0.1:1775772238.844535:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453f00. 00010000:00000010:0.1:1775772238.844540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8600. 00010000:00000010:0.1:1775772238.844543:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990100. 00010000:00000010:0.1:1775772238.844546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206b3da00. 00000080:00000010:0.1:1775772238.859188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd240. 00010000:00000010:2.1:1775772238.871317:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211933000. 00010000:00000010:2.1:1775772238.871327:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453800. 00010000:00000010:2.1:1775772238.871330:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8a00. 00010000:00000010:2.1:1775772238.871333:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202ae7c00. 00010000:00000010:2.1:1775772238.871336:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e102800. 00000080:00000010:2.1:1775772238.876536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772238.876572:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62184a1080. 00000080:00000010:2.1:1775772238.913281:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62184b2100. 00010000:00000010:2.1:1775772238.916259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297400. 00010000:00000010:2.1:1775772238.916269:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111800. 00010000:00000010:2.1:1775772238.916274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc48c00. 00010000:00000010:2.1:1775772238.916277:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3000. 00010000:00000010:2.1:1775772238.916280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00000080:00000010:2.1:1775772238.932885:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772238.932909:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192cc200. 00010000:00000010:2.1:1775772238.939199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297c00. 00010000:00000010:2.1:1775772238.939215:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120cab00. 00010000:00000010:2.1:1775772238.939219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89200. 00010000:00000010:2.1:1775772238.939223:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3000. 00010000:00000010:2.1:1775772238.939226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00000080:00000010:2.1:1775772238.960519:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:2.1:1775772238.960547:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192df380. 00010000:00000010:2.1:1775772238.967266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296c00. 00010000:00000010:2.1:1775772238.967282:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453500. 00010000:00000010:2.1:1775772238.967286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d9800. 00010000:00000010:2.1:1775772238.967290:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3000. 00010000:00000010:2.1:1775772238.967293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:2.1:1775772239.014401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211932e00. 00010000:00000010:2.1:1775772239.014416:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453200. 00010000:00000010:2.1:1775772239.014421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d8000. 00010000:00000010:2.1:1775772239.014425:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990f00. 00010000:00000010:2.1:1775772239.014429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206b3c200. 00000080:00000010:2.1:1775772239.024492:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:2.1:1775772239.024520:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62184b4200. 00010000:00000010:2.1:1775772239.053632:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211933600. 00010000:00000010:2.1:1775772239.053640:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111e00. 00010000:00000010:2.1:1775772239.053645:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc49c00. 00010000:00000010:2.1:1775772239.053649:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3000. 00010000:00000010:2.1:1775772239.053652:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00000080:00000010:2.1:1775772239.078108:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:2.1:1775772239.086300:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62184b0000. 00010000:00000010:2.1:1775772239.086309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211932000. 00010000:00000010:2.1:1775772239.086313:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453a00. 00010000:00000010:2.1:1775772239.086317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d9c00. 00010000:00000010:2.1:1775772239.086321:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3f00. 00010000:00000010:2.1:1775772239.086324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00000080:00000010:2.1:1775772239.091395:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00010000:00000010:0.1:1775772239.116185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211932a00. 00010000:00000010:0.1:1775772239.116193:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220453700. 00010000:00000010:0.1:1775772239.116197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c2d9400. 00010000:00000010:0.1:1775772239.116200:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990000. 00010000:00000010:0.1:1775772239.116203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206b3ce00. 00010000:00000010:2.1:1775772239.170296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46da00. 00010000:00000010:2.1:1775772239.170305:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111c00. 00010000:00000010:2.1:1775772239.170310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc49800. 00010000:00000010:2.1:1775772239.170313:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0000. 00010000:00000010:2.1:1775772239.170403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297200. 00000080:00000010:2.1:1775772239.183171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772239.188208:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62184b5ac0. 00010000:00000010:0.1:1775772239.194180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46c200. 00010000:00000010:0.1:1775772239.194188:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414e00. 00010000:00000010:0.1:1775772239.194193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ad200. 00010000:00000010:0.1:1775772239.194196:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3f00. 00010000:00000010:0.1:1775772239.194199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772239.236437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46ce00. 00010000:00000010:0.1:1775772239.236451:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111f00. 00010000:00000010:0.1:1775772239.236457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc48000. 00010000:00000010:0.1:1775772239.236461:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202ae7600. 00010000:00000010:0.1:1775772239.236463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e103a00. 00010000:00000010:2.1:1775772239.313211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46d400. 00010000:00000010:2.1:1775772239.313223:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414a00. 00010000:00000010:2.1:1775772239.313226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ada00. 00010000:00000010:2.1:1775772239.313229:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990a00. 00010000:00000010:2.1:1775772239.313231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206b3cc00. 00000080:00000010:2.1:1775772239.320164:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772239.320180:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62184b5280. 00010000:00000010:2.1:1775772239.332784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46ca00. 00010000:00000010:2.1:1775772239.332793:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414500. 00010000:00000010:2.1:1775772239.332796:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ace00. 00010000:00000010:2.1:1775772239.332799:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3000. 00010000:00000010:2.1:1775772239.332802:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00000080:00000010:2.1:1775772239.339621:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:2.1:1775772239.339651:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62184b3180. 00010000:00000010:2.1:1775772239.357148:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46c000. 00010000:00000010:2.1:1775772239.357159:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414800. 00010000:00000010:2.1:1775772239.357163:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ad600. 00010000:00000010:2.1:1775772239.357166:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3600. 00010000:00000010:2.1:1775772239.357170:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00000080:00000010:2.1:1775772239.362190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772239.366169:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62184b2940. 00010000:00000010:2.1:1775772239.388812:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46c600. 00010000:00000010:2.1:1775772239.388830:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414200. 00010000:00000010:2.1:1775772239.388835:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262acc00. 00010000:00000010:2.1:1775772239.388838:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3600. 00010000:00000010:2.1:1775772239.388842:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00000080:00000010:2.1:1775772239.398273:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772239.404935:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621875a100. 00010000:00000010:0.1:1775772239.427208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46d800. 00010000:00000010:0.1:1775772239.427222:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120cad00. 00010000:00000010:0.1:1775772239.427226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88600. 00010000:00000010:0.1:1775772239.427229:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3600. 00010000:00000010:0.1:1775772239.427232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:2.1:1775772239.478200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46c400. 00010000:00000010:2.1:1775772239.478219:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414900. 00010000:00000010:2.1:1775772239.478224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ad400. 00010000:00000010:2.1:1775772239.478229:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202ae7b00. 00010000:00000010:2.1:1775772239.478231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e102600. 00000080:00000010:2.1:1775772239.488161:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792c320. 00000080:00000010:2.1:1775772239.493181:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62184b6300. 00010000:00000010:2.1:1775772239.497190:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46d200. 00010000:00000010:2.1:1775772239.497195:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414100. 00010000:00000010:2.1:1775772239.497199:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ac000. 00010000:00000010:2.1:1775772239.497201:0:207604:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3000. 00010000:00000010:2.1:1775772239.497204:0:207604:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00000080:00000010:2.1:1775772239.506628:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:2.1:1775772239.506777:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62184b18c0. 00010000:00000010:2.1:1775772239.513191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46de00. 00010000:00000010:2.1:1775772239.513207:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414400. 00010000:00000010:2.1:1775772239.513212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ac600. 00010000:00000010:2.1:1775772239.513216:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3000. 00010000:00000010:2.1:1775772239.513220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00000080:00000010:2.1:1775772239.520170:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772239.527317:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f620950f380. 00010000:00000010:2.1:1775772239.531786:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46d000. 00010000:00000010:2.1:1775772239.531801:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414000. 00010000:00000010:2.1:1775772239.531806:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ac400. 00010000:00000010:2.1:1775772239.531809:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3000. 00010000:00000010:2.1:1775772239.531811:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00000080:00000010:2.1:1775772239.539237:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772239.543180:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209508840. 00010000:00000010:2.1:1775772239.550507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206765a00. 00010000:00000010:2.1:1775772239.550524:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414f00. 00010000:00000010:2.1:1775772239.550528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ac800. 00010000:00000010:2.1:1775772239.550531:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3000. 00010000:00000010:2.1:1775772239.550534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:2.1:1775772239.620196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206765600. 00010000:00000010:2.1:1775772239.620214:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210d8bb00. 00010000:00000010:2.1:1775772239.620220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262adc00. 00010000:00000010:2.1:1775772239.620224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00000080:00000010:2.1:1775772239.646174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:2.1:1775772239.646193:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f620950b180. 00010000:00000010:2.1:1775772239.681180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206764c00. 00010000:00000010:2.1:1775772239.681280:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205b00. 00010000:00000010:2.1:1775772239.681287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262ad000. 00010000:00000010:2.1:1775772239.681290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00000080:00000010:2.1:1775772239.681294:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:2.1:1775772239.681311:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f620950a940. 00010000:00000010:2.1:1775772239.681321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206765c00. 00010000:00000010:2.1:1775772239.681324:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205600. 00010000:00000010:2.1:1775772239.681327:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348be000. 00010000:00000010:2.1:1775772239.681330:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990800. 00010000:00000010:2.1:1775772239.681333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00000080:00000010:2.1:1775772239.693391:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772239.693418:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f620950ca40. 00010000:00000010:2.1:1775772239.711321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206765400. 00010000:00000010:2.1:1775772239.711337:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205000. 00010000:00000010:2.1:1775772239.711343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bea00. 00010000:00000010:2.1:1775772239.711346:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3000. 00010000:00000010:2.1:1775772239.711349:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00000080:00000010:2.1:1775772239.731228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772239.739251:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f620950e300. 00000080:00000010:2.1:1775772239.825173:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd540. 00000080:00000010:2.1:1775772239.832298:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226bc4a40. 00010000:00000010:2.1:1775772239.836251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206764600. 00010000:00000010:2.1:1775772239.836266:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205500. 00010000:00000010:2.1:1775772239.836271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348be800. 00010000:00000010:2.1:1775772239.836274:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3000. 00010000:00000010:2.1:1775772239.836277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00000080:00000010:2.1:1775772239.848570:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:2.1:1775772239.860815:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f620950a100. 00010000:00000010:0.1:1775772239.885002:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206765800. 00010000:00000010:0.1:1775772239.885018:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205100. 00010000:00000010:0.1:1775772239.885023:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348be400. 00010000:00000010:0.1:1775772239.885026:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3000. 00010000:00000010:0.1:1775772239.885030:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00000080:00000010:2.1:1775772239.913157:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f0e0. 00000080:00000010:2.1:1775772239.913175:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f620950c200. 00010000:00000010:2.1:1775772239.934816:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206764800. 00010000:00000010:2.1:1775772239.934832:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205400. 00010000:00000010:2.1:1775772239.934837:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348be600. 00010000:00000010:2.1:1775772239.934840:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3100. 00010000:00000010:2.1:1775772239.934843:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00000080:00000010:0.1:1775772239.963247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:0.1:1775772239.963271:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209508000. 00010000:00000010:2.1:1775772240.015878:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206765200. 00010000:00000010:2.1:1775772240.015897:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205300. 00010000:00000010:2.1:1775772240.015902:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bfe00. 00010000:00000010:2.1:1775772240.015906:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990800. 00010000:00000010:2.1:1775772240.015908:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206b3dc00. 00000080:00000010:2.1:1775772240.027389:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f280. 00000080:00000010:2.1:1775772240.027420:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209509080. 00010000:00000010:2.1:1775772240.036446:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206765e00. 00010000:00000010:2.1:1775772240.036465:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205200. 00010000:00000010:2.1:1775772240.036471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bfa00. 00010000:00000010:2.1:1775772240.036475:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3100. 00010000:00000010:2.1:1775772240.036478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00000080:00000010:2.1:1775772240.053127:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772240.056165:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f620950dac0. 00000080:00000010:0.1:1775772240.079547:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:0.1:1775772240.079567:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185639c0. 00010000:00000010:0.1:1775772240.084317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206764200. 00010000:00000010:0.1:1775772240.084332:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111d00. 00010000:00000010:0.1:1775772240.084337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc49200. 00010000:00000010:0.1:1775772240.084341:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0600. 00010000:00000010:0.1:1775772240.084344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297200. 00000080:00000010:0.1:1775772240.098189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f800. 00000080:00000010:0.1:1775772240.102170:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621875d280. 00010000:00000010:0.1:1775772240.109393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca72000. 00010000:00000010:0.1:1775772240.109498:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205900. 00010000:00000010:0.1:1775772240.109504:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348be200. 00010000:00000010:0.1:1775772240.109508:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0600. 00010000:00000010:0.1:1775772240.109511:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297200. 00000080:00000010:0.1:1775772240.124373:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f800. 00000080:00000010:0.1:1775772240.124405:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6218565280. 00010000:00000010:0.1:1775772240.138299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca72600. 00010000:00000010:0.1:1775772240.138313:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205700. 00010000:00000010:0.1:1775772240.138316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bfc00. 00010000:00000010:0.1:1775772240.138319:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0600. 00010000:00000010:0.1:1775772240.138321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297200. 00000080:00000010:0.1:1775772240.158298:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f800. 00000080:00000010:0.1:1775772240.167571:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6218563180. 00010000:00000010:2.1:1775772240.171669:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca73800. 00010000:00000010:2.1:1775772240.171679:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111600. 00010000:00000010:2.1:1775772240.171684:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc49400. 00010000:00000010:2.1:1775772240.171687:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0600. 00010000:00000010:2.1:1775772240.171690:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297200. 00000080:00000010:2.1:1775772240.177179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:2.1:1775772240.177208:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621875b9c0. 00010000:00000010:2.1:1775772240.190200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca72800. 00010000:00000010:2.1:1775772240.190215:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205800. 00010000:00000010:2.1:1775772240.190220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bec00. 00010000:00000010:2.1:1775772240.190223:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3a00. 00010000:00000010:2.1:1775772240.190226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00000080:00000010:0.1:1775772240.204102:0:207604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:0.1:1775772240.204123:0:207604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6218562940. 00000080:00000010:2.1:1775772240.237581:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6218564a40. 00010000:00000010:2.1:1775772240.241338:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca73200. 00010000:00000010:2.1:1775772240.241350:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9a00. 00010000:00000010:2.1:1775772240.241355:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bf600. 00010000:00000010:2.1:1775772240.241359:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3a00. 00010000:00000010:2.1:1775772240.241362:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00000080:00000010:2.1:1775772240.264281:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:2.1:1775772240.264308:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6218566300. 00010000:00000010:2.1:1775772240.267323:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca73e00. 00010000:00000010:2.1:1775772240.267340:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0100. 00010000:00000010:2.1:1775772240.267345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89000. 00010000:00000010:2.1:1775772240.267348:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3a00. 00010000:00000010:2.1:1775772240.267351:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00000080:00000010:2.1:1775772240.277353:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775772240.277385:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192d8000. 00010000:00000010:2.1:1775772240.327184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca73a00. 00010000:00000010:2.1:1775772240.327192:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9900. 00010000:00000010:2.1:1775772240.327196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bf800. 00010000:00000010:2.1:1775772240.327199:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990a00. 00010000:00000010:2.1:1775772240.327202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206b3d600. 00000080:00000010:0.1:1775772240.332262:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185618c0. 00000080:00000010:2.1:1775772240.332327:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd020. 00010000:00000010:2.1:1775772240.423403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca73600. 00010000:00000010:2.1:1775772240.423418:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0a00. 00010000:00000010:2.1:1775772240.423422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88800. 00010000:00000010:2.1:1775772240.423426:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202ae7b00. 00010000:00000010:2.1:1775772240.423428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be4c00. 00000080:00000010:2.1:1775772240.468265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:2.1:1775772240.468290:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6218564200. 00000080:00000010:0.1:1775772240.588171:0:203109:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2760. 00000080:00000010:0.1:1775772240.588188:0:203109:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4d00. 00000080:00000010:0.1:1775772240.588225:0:203109:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a20c0. 00000080:00000010:0.1:1775772240.588232:0:203109:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2440. 00000080:00000010:0.1:1775772240.588259:0:203109:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2c60. 00000080:00000010:0.1:1775772240.596526:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2740. 00000080:00000010:0.1:1775772240.596571:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0ea40. 00000080:00000010:0.1:1775772240.599166:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2400. 00000080:00000010:0.1:1775772240.599238:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2140. 00000080:00000010:0.1:1775772240.599247:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4420. 00000080:00000010:0.1:1775772240.599254:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd360. 00000080:00000010:0.1:1775772240.599259:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2ba0. 00000080:00000010:0.1:1775772240.599263:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0efe0. 00000080:00000010:0.1:1775772240.599288:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdb60. 00010000:00000010:2.1:1775772240.606157:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb100. 00010000:00000010:2.1:1775772240.606164:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4b600. 00010000:00000010:2.1:1775772240.606168:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb900. 00010000:00000010:2.1:1775772240.606171:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628be00. 00010000:00000010:2.1:1775772240.606174:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210a9e700. 00010000:00000010:2.1:1775772240.606178:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101124c00. 00010000:00000010:2.1:1775772240.606181:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210a9e600. 00010000:00000010:2.1:1775772240.606184:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101124e00. 00010000:00000010:2.1:1775772240.606186:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210a9e400. 00010000:00000010:2.1:1775772240.606189:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101125a00. 00010000:00000010:2.1:1775772240.606194:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e332300. 00010000:00000010:2.1:1775772240.606197:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27200. 00010000:00000010:0.1:1775772240.606279:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9ce00. 00010000:00000010:2.1:1775772240.606289:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210a9ee00. 00010000:00000010:0.1:1775772240.606296:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25600. 00010000:00000010:2.1:1775772240.606301:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101125e00. 00010000:00000010:0.1:1775772240.606314:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6f700. 00010000:00000010:2.1:1775772240.606315:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e332a00. 00010000:00000010:0.1:1775772240.606335:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45800. 00010000:00000010:2.1:1775772240.606335:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27800. 00010000:00000010:2.1:1775772240.606346:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207da4000. 00010000:00000010:0.1:1775772240.606350:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44a00. 00010000:00000010:2.1:1775772240.606352:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226005600. 00010000:00000010:0.1:1775772240.606362:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206551c00. 00010000:00000010:2.1:1775772240.606363:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e332800. 00010000:00000010:0.1:1775772240.606377:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72e00. 00010000:00000010:2.1:1775772240.606378:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd27600. 00010000:00000010:0.1:1775772240.606391:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388ea00. 00010000:00000010:0.1:1775772240.606408:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73400. 00010000:00000010:0.1:1775772240.606424:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388eb00. 00010000:00000010:0.1:1775772240.606438:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec000. 00010000:00000010:0.1:1775772240.606449:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345edc00. 00010000:00000010:0.1:1775772240.606460:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388ee00. 00010000:00000010:0.1:1775772240.606470:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec600. 00010000:00000010:0.1:1775772240.606489:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec200. 00010000:00000010:0.1:1775772240.606503:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6f300. 00010000:00000010:0.1:1775772240.606520:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101127000. 00010000:00000010:0.1:1775772240.606549:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388ed00. 00010000:00000010:0.1:1775772240.606563:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec400. 00010000:00000010:0.1:1775772240.606578:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ec800. 00010000:00000010:0.1:1775772240.606589:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e300. 00010000:00000010:0.1:1775772240.606608:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101126e00. 00010000:00000010:0.1:1775772240.606619:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6f900. 00010000:00000010:0.1:1775772240.606633:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45e00. 00010000:00000010:0.1:1775772240.606645:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9cc00. 00010000:00000010:0.1:1775772240.606658:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25400. 00010000:00000010:0.1:1775772240.606684:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e200. 00010000:00000010:0.1:1775772240.606696:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ece00. 00010000:00000010:0.1:1775772240.606707:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9c100. 00010000:00000010:0.1:1775772240.606716:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24000. 00010000:00000010:0.1:1775772240.606729:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e800. 00010000:00000010:0.1:1775772240.606739:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345eca00. 00010000:00000010:0.1:1775772240.606747:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9c400. 00010000:00000010:0.1:1775772240.606754:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24400. 00010000:00000010:0.1:1775772240.606758:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6f800. 00010000:00000010:0.1:1775772240.608258:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101126800. 00010000:00000010:0.1:1775772240.608262:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ed000. 00010000:00000010:0.1:1775772240.608266:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9c800. 00010000:00000010:0.1:1775772240.608269:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25200. 00010000:00000010:0.1:1775772240.608275:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345eda00. 00010000:00000010:0.1:1775772240.608278:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e500. 00010000:00000010:0.1:1775772240.608282:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ede00. 00010000:00000010:0.1:1775772240.608288:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25e00. 00010000:00000010:0.1:1775772240.608294:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9c700. 00010000:00000010:0.1:1775772240.608301:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24600. 00010000:00000010:0.1:1775772240.608307:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae2b00. 00010000:00000010:0.1:1775772240.608318:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335e00. 00010000:00000010:0.1:1775772240.608323:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae2300. 00010000:00000010:0.1:1775772240.608329:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233a2d800. 00010000:00000010:0.1:1775772240.608341:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5f00. 00010000:00000010:0.1:1775772240.608348:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7e00. 00010000:00000010:0.1:1775772240.608356:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c25fc00. 00010000:00000010:0.1:1775772240.608364:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101122000. 00010000:00000010:0.1:1775772240.608368:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c25f800. 00010000:00000010:0.1:1775772240.608371:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101122c00. 00010000:00000010:0.1:1775772240.608373:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c25f500. 00010000:00000010:0.1:1775772240.608377:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101123400. 00010000:00000010:0.1:1775772240.608385:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae2200. 00010000:00000010:0.1:1775772240.608411:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233a2cc00. 00010000:00000010:0.1:1775772240.608422:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e480f00. 00010000:00000010:0.1:1775772240.608429:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233a2ce00. 00010000:00000010:0.1:1775772240.608433:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e480a00. 00010000:00000010:0.1:1775772240.608441:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233a2d600. 00010000:00000010:0.1:1775772240.608452:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e480d00. 00010000:00000010:0.1:1775772240.608460:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101123600. 00010000:00000010:0.1:1775772240.608471:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62039cd700. 00010000:00000010:0.1:1775772240.608478:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101123a00. 00010000:00000010:0.1:1775772240.608484:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e480200. 00010000:00000010:0.1:1775772240.608495:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167e400. 00010000:00000010:0.1:1775772240.608505:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62039cdc00. 00010000:00000010:0.1:1775772240.608512:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200031200. 00010000:00000010:0.1:1775772240.608521:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e480900. 00010000:00000010:0.1:1775772240.608526:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167e600. 00010000:00000010:0.1:1775772240.608547:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62039cd900. 00010000:00000010:0.1:1775772240.608553:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200030a00. 00010000:00000010:0.1:1775772240.608560:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e6c5100. 00010000:00000010:0.1:1775772240.608563:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167f400. 00010000:00000010:0.1:1775772240.608567:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e6c5a00. 00010000:00000010:0.1:1775772240.608570:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167ee00. 00010000:00000010:0.1:1775772240.608574:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5e00. 00010000:00000010:0.1:1775772240.608578:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6e00. 00010000:00000010:0.1:1775772240.608581:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5000. 00010000:00000010:0.1:1775772240.608584:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6600. 00010000:00000010:0.1:1775772240.608587:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e6c5b00. 00010000:00000010:0.1:1775772240.608589:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200031800. 00010000:00000010:0.1:1775772240.608592:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e6c5f00. 00010000:00000010:0.1:1775772240.608596:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dde00. 00010000:00000010:0.1:1775772240.608757:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e6c5600. 00010000:00000010:0.1:1775772240.608842:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200031e00. 00010000:00000010:0.1:1775772240.609105:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5600. 00010000:00000010:0.1:1775772240.609123:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6000. 00010000:00000010:0.1:1775772240.609129:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62039cd500. 00010000:00000010:0.1:1775772240.609170:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201528800. 00010000:00000010:0.1:1775772240.609179:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e6c5700. 00010000:00000010:0.1:1775772240.609186:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dd200. 00010000:00000010:0.1:1775772240.609193:0:2412:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e6c5500. 00010000:00000010:0.1:1775772240.609200:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dd000. 00010000:00000010:2.1:1775772240.613013:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c25f600. 00010000:00000010:2.1:1775772240.613032:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101124a00. 00010000:00000010:2.1:1775772240.613036:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64100. 00010000:00000010:2.1:1775772240.613039:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101124600. 00010000:00000010:2.1:1775772240.613042:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e332900. 00010000:00000010:2.1:1775772240.613047:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26400. 00010000:00000010:2.1:1775772240.613051:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207da4b00. 00010000:00000010:2.1:1775772240.613054:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628bc00. 00010000:00000010:2.1:1775772240.613057:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111200. 00010000:00000010:2.1:1775772240.613060:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101124800. 00010000:00000010:2.1:1775772240.613065:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210414600. 00010000:00000010:2.1:1775772240.613068:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628b400. 00010000:00000010:2.1:1775772240.613070:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207da4900. 00010000:00000010:2.1:1775772240.613073:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628a600. 00010000:00000010:2.1:1775772240.613076:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207da4600. 00010000:00000010:2.1:1775772240.613078:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628aa00. 00010000:00000010:2.1:1775772240.613081:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bed700. 00010000:00000010:2.1:1775772240.613087:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226005200. 00010000:00000010:2.1:1775772240.613093:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bed100. 00010000:00000010:2.1:1775772240.613099:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101127a00. 00010000:00000010:2.1:1775772240.613107:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0c00. 00010000:00000010:2.1:1775772240.613115:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296e00. 00010000:00000010:2.1:1775772240.613123:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0400. 00010000:00000010:2.1:1775772240.613130:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297a00. 00010000:00000010:2.1:1775772240.613154:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9000. 00010000:00000010:2.1:1775772240.613158:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628b000. 00010000:00000010:2.1:1775772240.613166:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9600. 00010000:00000010:2.1:1775772240.613169:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628b600. 00010000:00000010:2.1:1775772240.613175:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9b00. 00010000:00000010:2.1:1775772240.613180:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628a200. 00010000:00000010:2.1:1775772240.613200:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9400. 00010000:00000010:2.1:1775772240.613208:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622628b200. 00010000:00000010:2.1:1775772240.613223:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3e00. 00010000:00000010:2.1:1775772240.613240:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9200. 00010000:00000010:2.1:1775772240.613258:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b89d00. 00010000:00000010:2.1:1775772240.613277:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1da00. 00010000:00000010:2.1:1775772240.613292:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e40ba00. 00010000:00000010:2.1:1775772240.613300:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1de00. 00010000:00000010:2.1:1775772240.613307:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e40b800. 00010000:00000010:2.1:1775772240.613316:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1c000. 00010000:00000010:2.1:1775772240.613326:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fbf00. 00010000:00000010:2.1:1775772240.613334:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206b3d400. 00010000:00000010:2.1:1775772240.613345:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fb400. 00010000:00000010:2.1:1775772240.613362:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4a000. 00010000:00000010:2.1:1775772240.613382:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211364100. 00010000:00000010:2.1:1775772240.613399:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4ba00. 00010000:00000010:2.1:1775772240.613408:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb800. 00010000:00000010:2.1:1775772240.613411:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4b400. 00010000:00000010:2.1:1775772240.613414:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211939c00. 00010000:00000010:2.1:1775772240.613418:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8800. 00010000:00000010:2.1:1775772240.613421:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211939400. 00010000:00000010:2.1:1775772240.613423:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:2.1:1775772240.613426:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211939700. 00010000:00000010:2.1:1775772240.613428:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cd26600. 00010000:00000010:2.1:1775772240.613435:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211939f00. 00010000:00000010:2.1:1775772240.613442:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1ca00. 00010000:00000010:2.1:1775772240.613449:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b89a00. 00010000:00000010:2.1:1775772240.613457:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1dc00. 00010000:00000010:2.1:1775772240.613465:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b89000. 00010000:00000010:2.1:1775772240.613478:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1d600. 00010000:00000010:2.1:1775772240.613489:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b89300. 00010000:00000010:2.1:1775772240.613497:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1c200. 00010000:00000010:2.1:1775772240.613506:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e40bb00. 00010000:00000010:2.1:1775772240.613513:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1c800. 00010000:00000010:2.1:1775772240.613522:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e40bd00. 00010000:00000010:2.1:1775772240.613529:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1c600. 00010000:00000010:2.1:1775772240.616617:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990c00. 00010000:00000010:2.1:1775772240.616628:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206b3c400. 00010000:00000010:2.1:1775772240.616631:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990b00. 00010000:00000010:2.1:1775772240.616635:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206b3d800. 00010000:00000010:2.1:1775772240.617167:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fbb00. 00010000:00000010:2.1:1775772240.617173:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206b3c000. 00010000:00000010:2.1:1775772240.617178:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fb100. 00010000:00000010:2.1:1775772240.617184:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4a600. 00010000:00000010:2.1:1775772240.617188:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fbc00. 00010000:00000010:2.1:1775772240.617197:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4a800. 00010000:00000010:2.1:1775772240.617201:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211364400. 00010000:00000010:2.1:1775772240.617204:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4b200. 00010000:00000010:2.1:1775772240.617207:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211364700. 00010000:00000010:2.1:1775772240.617210:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4a200. 00010000:00000010:2.1:1775772240.617214:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcbb00. 00010000:00000010:2.1:1775772240.617217:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4ac00. 00010000:00000010:2.1:1775772240.618289:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e480b00. 00010000:00000010:2.1:1775772240.618300:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dc800. 00010000:00000010:2.1:1775772240.618310:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e480300. 00010000:00000010:2.1:1775772240.618321:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167fa00. 00010000:00000010:2.1:1775772240.618331:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62039cda00. 00010000:00000010:2.1:1775772240.618341:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200030000. 00010000:00000010:2.1:1775772240.618351:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e480c00. 00010000:00000010:2.1:1775772240.618360:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167ec00. 00010000:00000010:2.1:1775772240.618367:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e480e00. 00010000:00000010:2.1:1775772240.618377:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167e000. 00010000:00000010:2.1:1775772240.618387:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e480500. 00010000:00000010:2.1:1775772240.618399:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167fe00. 00010000:00000010:2.1:1775772240.618410:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e6c5300. 00010000:00000010:2.1:1775772240.618416:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167f000. 00010000:00000010:2.1:1775772240.618420:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62039cd200. 00010000:00000010:2.1:1775772240.618424:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200030800. 00010000:00000010:2.1:1775772240.618427:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e6c5900. 00010000:00000010:2.1:1775772240.618431:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167ea00. 00010000:00000010:2.1:1775772240.618434:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167f800. 00010000:00000010:2.1:1775772240.618439:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e6c5000. 00010000:00000010:2.1:1775772240.618447:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167e200. 00010000:00000010:2.1:1775772240.618458:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e6c5400. 00010000:00000010:2.1:1775772240.618468:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6800. 00010000:00000010:2.1:1775772240.618475:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5c00. 00010000:00000010:2.1:1775772240.618480:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6200. 00010000:00000010:2.1:1775772240.618483:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62039cdb00. 00010000:00000010:2.1:1775772240.618487:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200030400. 00010000:00000010:2.1:1775772240.618494:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200030600. 00010000:00000010:2.1:1775772240.618497:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e6c5200. 00010000:00000010:2.1:1775772240.618501:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167fc00. 00010000:00000010:2.1:1775772240.618508:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e6c5e00. 00010000:00000010:2.1:1775772240.618513:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dc000. 00010000:00000010:2.1:1775772240.618519:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62039cd000. 00010000:00000010:2.1:1775772240.618528:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200031000. 00010000:00000010:2.1:1775772240.618534:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5400. 00010000:00000010:2.1:1775772240.618538:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7a00. 00010000:00000010:2.1:1775772240.618545:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5500. 00010000:00000010:2.1:1775772240.618557:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45a00. 00010000:00000010:2.1:1775772240.618566:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e6c5d00. 00010000:00000010:2.1:1775772240.618573:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dc400. 00010000:00000010:2.1:1775772240.618578:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e6c5800. 00010000:00000010:2.1:1775772240.618587:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45c00. 00010000:00000010:2.1:1775772240.618593:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201528000. 00010000:00000010:2.1:1775772240.618602:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e6c5c00. 00010000:00000010:2.1:1775772240.618606:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dc600. 00010000:00000010:2.1:1775772240.618614:0:207610:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e14de00. 00010000:00000010:2.1:1775772240.618621:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fce8800. 00010000:00000010:2.1:1775772240.618631:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fce9000. 00010000:00000010:0.1:1775772240.659066:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621022fc00. 00010000:00000010:0.1:1775772240.659086:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234916a00. 00010000:00000010:0.1:1775772240.659091:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621022ff00. 00010000:00000010:0.1:1775772240.659093:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234916c00. 00010000:00000010:0.1:1775772240.659096:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621022f600. 00010000:00000010:0.1:1775772240.659099:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234917c00. 00010000:00000010:0.1:1775772240.659102:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62421f5000. 00010000:00000010:0.1:1775772240.659114:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234916e00. 00010000:00000010:0.1:1775772240.659118:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64600. 00010000:00000010:0.1:1775772240.659121:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101152200. 00010000:00000010:0.1:1775772240.659126:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62421f5c00. 00010000:00000010:0.1:1775772240.659130:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb05e00. 00010000:00000010:0.1:1775772240.659147:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62421f5b00. 00010000:00000010:0.1:1775772240.659151:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb05200. 00010000:00000010:0.1:1775772240.659154:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62421f5a00. 00010000:00000010:0.1:1775772240.659157:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb04800. 00010000:00000010:0.1:1775772240.659163:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62347fff00. 00010000:00000010:0.1:1775772240.659167:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb04a00. 00010000:00000010:0.1:1775772240.659172:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62347ff500. 00010000:00000010:0.1:1775772240.659176:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb05400. 00010000:00000010:0.1:1775772240.659181:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62347ff000. 00010000:00000010:0.1:1775772240.659184:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb05c00. 00010000:00000010:0.1:1775772240.659188:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5b00. 00010000:00000010:0.1:1775772240.659192:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7000. 00010000:00000010:0.1:1775772240.659195:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62347ff300. 00010000:00000010:0.1:1775772240.659200:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb05600. 00010000:00000010:0.1:1775772240.659203:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62039cd100. 00010000:00000010:0.1:1775772240.659206:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101153600. 00010000:00000010:0.1:1775772240.659209:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62347ffe00. 00010000:00000010:0.1:1775772240.659212:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb04200. 00010000:00000010:0.1:1775772240.659216:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62347ffa00. 00010000:00000010:0.1:1775772240.659219:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e40c800. 00010000:00000010:0.1:1775772240.659237:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6212fbb500. 00010000:00000010:0.1:1775772240.659241:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101153a00. 00010000:00000010:0.1:1775772240.659245:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c25ff00. 00010000:00000010:0.1:1775772240.659249:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101123800. 00010000:00000010:0.1:1775772240.659252:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c25f100. 00010000:00000010:0.1:1775772240.659254:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101153e00. 00010000:00000010:0.1:1775772240.659257:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c25fb00. 00010000:00000010:0.1:1775772240.659260:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e40d400. 00010000:00000010:0.1:1775772240.659264:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6212fbb300. 00010000:00000010:0.1:1775772240.659267:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e40da00. 00010000:00000010:0.1:1775772240.659270:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6212fbb800. 00010000:00000010:0.1:1775772240.659274:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e334a00. 00010000:00000010:0.1:1775772240.659279:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae2700. 00010000:00000010:0.1:1775772240.659282:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335600. 00010000:00000010:0.1:1775772240.659285:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c25fa00. 00010000:00000010:0.1:1775772240.659288:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101122800. 00010000:00000010:0.1:1775772240.659292:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6900. 00010000:00000010:0.1:1775772240.659295:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101126000. 00010000:00000010:0.1:1775772240.659299:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae2600. 00010000:00000010:0.1:1775772240.659303:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335000. 00010000:00000010:2.1:1775772240.668193:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e14d900. 00010000:00000010:2.1:1775772240.668201:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101127400. 00010000:00000010:2.1:1775772240.668206:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9c500. 00010000:00000010:2.1:1775772240.668211:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201528c00. 00010000:00000010:2.1:1775772240.668217:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e14d300. 00010000:00000010:2.1:1775772240.668223:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fce8e00. 00010000:00000010:2.1:1775772240.668227:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fce8200. 00010000:00000010:2.1:1775772240.668231:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e14db00. 00010000:00000010:2.1:1775772240.668235:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fce9a00. 00010000:00000010:2.1:1775772240.668238:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e14d000. 00010000:00000010:2.1:1775772240.668240:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fce9600. 00010000:00000010:2.1:1775772240.668242:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9ca00. 00010000:00000010:2.1:1775772240.668245:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201529400. 00010000:00000010:2.1:1775772240.668249:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e14d700. 00010000:00000010:2.1:1775772240.668253:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fce9e00. 00010000:00000010:2.1:1775772240.668256:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44800. 00010000:00000010:2.1:1775772240.668275:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e14df00. 00010000:00000010:2.1:1775772240.668279:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fce8400. 00010000:00000010:2.1:1775772240.668282:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201529600. 00010000:00000010:2.1:1775772240.668285:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9cb00. 00010000:00000010:2.1:1775772240.668288:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201529800. 00010000:00000010:2.1:1775772240.668290:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9c000. 00010000:00000010:2.1:1775772240.668293:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201528a00. 00010000:00000010:2.1:1775772240.668296:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de9cd00. 00010000:00000010:2.1:1775772240.668301:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24e00. 00010000:00000010:2.1:1775772240.668304:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e14d500. 00010000:00000010:2.1:1775772240.668306:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fce8000. 00010000:00000010:2.1:1775772240.668309:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb25800. 00010000:00000010:2.1:1775772240.668312:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e14d400. 00010000:00000010:2.1:1775772240.668315:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fce8c00. 00010000:00000010:2.1:1775772240.668318:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fce9800. 00010000:00000010:2.1:1775772240.668320:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e14d600. 00010000:00000010:2.1:1775772240.668323:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb24a00. 00010000:00000010:2.1:1775772240.668326:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206551200. 00010000:00000010:2.1:1775772240.668330:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72200. 00010000:00000010:2.1:1775772240.668334:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e14d200. 00010000:00000010:2.1:1775772240.668337:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fce9c00. 00010000:00000010:2.1:1775772240.668340:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206551300. 00010000:00000010:2.1:1775772240.668342:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72a00. 00010000:00000010:2.1:1775772240.668345:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206551b00. 00010000:00000010:2.1:1775772240.668348:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72400. 00010000:00000010:2.1:1775772240.668350:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72600. 00010000:00000010:2.1:1775772240.668353:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206551f00. 00010000:00000010:2.1:1775772240.668356:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73c00. 00010000:00000010:2.1:1775772240.668360:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206551d00. 00010000:00000010:2.1:1775772240.668363:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73a00. 00000080:00000010:2.1:1775772240.702173:0:207454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f097380. 00000080:00000010:2.1:1775772240.702201:0:207454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192c8840. 00010000:00000010:0.1:1775772240.702204:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203a46300. 00000080:00000010:2.1:1775772240.702208:0:207454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192cca40. 00010000:00000010:0.1:1775772240.702227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85400. 00000080:00000010:2.1:1775772240.702229:0:207454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f090000. 00000080:00000010:2.1:1775772240.702240:0:207454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f094200. 00010000:00000010:0.1:1775772240.702250:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203a46600. 00000080:00000010:2.1:1775772240.702253:0:207454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f094a40. 00010000:00000010:0.1:1775772240.702258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84c00. 00000080:00000010:2.1:1775772240.702266:0:207454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226bc7380. 00010000:00000010:0.1:1775772240.702277:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621022fb00. 00000080:00000010:2.1:1775772240.702279:0:207454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f093180. 00010000:00000010:0.1:1775772240.702288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234917000. 00000080:00000010:2.1:1775772240.702290:0:207454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226bc5ac0. 00010000:00000010:0.1:1775772240.702298:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203a46000. 00010000:00000010:0.1:1775772240.702303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234917e00. 00010000:00000010:0.1:1775772240.702310:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6000. 00000080:00000010:2.1:1775772240.702316:0:207454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621efe0840. 00010000:00000010:0.1:1775772240.702318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226005000. 00010000:00000010:0.1:1775772240.702326:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6b00. 00010000:00000010:0.1:1775772240.702336:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226004400. 00000080:00000010:2.1:1775772240.702337:0:207454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621efe4a40. 00000080:00000010:2.1:1775772240.702345:0:207454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621efe4200. 00000080:00000010:2.1:1775772240.719867:0:188720:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226bc0000. 00000080:00000010:2.1:1775772240.719882:0:188720:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f096300. 00000080:00000010:2.1:1775772240.719887:0:188720:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f091080. 00000080:00000010:2.1:1775772240.719895:0:188720:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6219144a40. 00000080:00000010:2.1:1775772240.719902:0:188720:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226830840. 00000080:00000010:2.1:1775772240.719909:0:188720:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209522940. 00000080:00000010:2.1:1775772240.719915:0:188720:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621906d280. 00000080:00000010:2.1:1775772240.719919:0:188720:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209520000. 00000080:00000010:2.1:1775772240.720101:0:188720:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621906b180. 00000080:00000010:2.1:1775772240.720109:0:188720:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202263180. 00000080:00000010:2.1:1775772240.720131:0:188720:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62268318c0. 00000080:00000010:2.1:1775772240.720164:0:188720:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62268f4a40. 00000080:00000010:2.1:1775772240.720171:0:188720:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f022100. 00000080:00000010:2.1:1775772240.721891:0:188720:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62268f2100. 00000080:00000010:2.1:1775772240.721907:0:188720:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226831080. 00000080:00000010:2.1:1775772240.721913:0:188720:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226835ac0. 00000080:00000010:2.1:1775772240.721919:0:188720:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226bc39c0. 00000080:00000010:2.1:1775772240.721940:0:188720:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209525ac0. 00000080:00000010:2.1:1775772240.721995:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192ca100. 00000080:00000010:2.1:1775772240.722001:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621efe5280. 00000080:00000010:2.1:1775772240.722007:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621efe5ac0. 00000080:00000010:2.1:1775772240.722014:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621efe2940. 00000080:00000010:2.1:1775772240.722023:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f092940. 00000080:00000010:2.1:1775772240.722047:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226bc1080. 00000080:00000010:2.1:1775772240.722052:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f0918c0. 00000080:00000010:2.1:1775772240.722056:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f096b40. 00000080:00000010:2.1:1775772240.722062:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192ca940. 00000080:00000010:2.1:1775772240.722066:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f0939c0. 00000080:00000010:2.1:1775772240.722078:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f090840. 00000080:00000010:2.1:1775772240.722083:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f620216e300. 00000080:00000010:2.1:1775772240.722087:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f620216f380. 00000080:00000010:2.1:1775772240.722091:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202168840. 00000080:00000010:2.1:1775772240.722112:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185c5ac0. 00000080:00000010:2.1:1775772240.722127:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6219147380. 00000080:00000010:2.1:1775772240.722132:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62191418c0. 00000080:00000010:2.1:1775772240.722155:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6219142940. 00000080:00000010:2.1:1775772240.722160:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6219146300. 00000080:00000010:2.1:1775772240.722164:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226830000. 00000080:00000010:2.1:1775772240.722179:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6226833180. 00000080:00000010:2.1:1775772240.722184:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6219143180. 00000080:00000010:2.1:1775772240.722188:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6219140840. 00000080:00000010:2.1:1775772240.722192:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6219144200. 00000080:00000010:2.1:1775772240.722196:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62268339c0. 00000080:00000010:2.1:1775772240.722218:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185c7380. 00000080:00000010:2.1:1775772240.722223:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185c1080. 00000080:00000010:2.1:1775772240.722228:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62268f4200. 00000080:00000010:2.1:1775772240.722232:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62268f0000. 00000080:00000010:2.1:1775772240.722235:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62268f18c0. 00000080:00000010:2.1:1775772240.722248:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62268f2940. 00000080:00000010:2.1:1775772240.722253:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62268f5ac0. 00000080:00000010:2.1:1775772240.722257:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62268f5280. 00000080:00000010:2.1:1775772240.722261:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185c2100. 00000080:00000010:2.1:1775772240.722264:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185c39c0. 00000080:00000010:2.1:1775772240.722275:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185c3180. 00000080:00000010:2.1:1775772240.722279:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62185c4a40. 00000080:00000010:2.1:1775772240.722282:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621efe6300. 00000080:00000010:2.1:1775772240.722286:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202214a40. 00000080:00000010:2.1:1775772240.722291:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62022139c0. 00000080:00000010:2.1:1775772240.722303:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202213180. 00000080:00000010:2.1:1775772240.722308:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62022118c0. 00000080:00000010:2.1:1775772240.722312:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202212100. 00000080:00000010:2.1:1775772240.722317:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202210000. 00000080:00000010:2.1:1775772240.722320:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6202214200. 00000080:00000010:2.1:1775772240.722335:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209522100. 00000080:00000010:2.1:1775772240.722339:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209526300. 00000080:00000010:2.1:1775772240.722344:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f024200. 00000080:00000010:2.1:1775772240.722348:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209525280. 00000080:00000010:2.1:1775772240.722351:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209523180. 00000080:00000010:2.1:1775772240.722634:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209524a40. 00000080:00000010:2.1:1775772240.722644:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f6209520840. 00000080:00000010:2.1:1775772240.722649:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f020000. 00000080:00000010:2.1:1775772240.722653:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621f026b40. 00000080:00000010:2.1:1775772240.722658:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62095239c0. 00010000:00000010:2.1:1775772251.598165:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990500. 00010000:00000010:2.1:1775772251.598188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206b3dc00. 00000080:00000010:2.1:1775772251.653261:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e460. 00010000:00000010:2.1:1775772251.871201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00000080:00000010:2.1:1775772251.871222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e460. 00010000:00000010:2.1:1775772251.897684:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9a00. 00000080:00000010:2.1:1775772251.897706:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e1e0. 00010000:00000010:2.1:1775772251.907198:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3700. 00010000:00000010:0.1:1775772251.914455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:2.1:1775772251.922191:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3e00. 00010000:00000010:2.1:1775772251.922207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:2.1:1775772251.982469:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3700. 00010000:00000010:2.1:1775772251.982484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9a00. 00000080:00000010:2.1:1775772252.020187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e2e0. 00010000:00000010:2.1:1775772271.519192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be5c00. 00010000:00000010:2.1:1775772271.784184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be4e00. 00010000:00000010:0.1:1775772271.798464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772271.806176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297800. 00010000:00000010:0.1:1775772271.828852:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be4c00. 00010000:00000010:2.1:1775772271.836252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84400. 00010000:00000010:0.1:1775772271.895207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:2.1:1775772271.899188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772271.899242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:0.1:1775772271.916309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:0.1:1775772271.937274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297800. 00010000:00000010:0.1:1775772271.966117:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297800. 00010000:00000010:0.1:1775772271.971401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772271.997198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772272.070273:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772272.095298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296e00. 00010000:00000010:0.1:1775772272.153197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296c00. 00010000:00000010:0.1:1775772272.175955:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296a00. 00010000:00000010:0.1:1775772272.189654:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772272.339637:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772272.428886:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772272.442180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772272.451233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be4a00. 00010000:00000010:0.1:1775772272.451240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772272.479788:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be4e00. 00010000:00000010:0.1:1775772272.500654:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772272.541181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772272.574852:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772272.596473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be4200. 00010000:00000010:0.1:1775772272.615188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772272.645181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:2.1:1775772272.671204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772272.715183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772272.759206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772272.820682:0:207609:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772272.828091:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:2.1:1775772272.915930:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296800. 00010000:00000010:0.1:1775772272.950653:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297e00. 00010000:00000010:0.1:1775772272.975519:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772273.009190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772273.050263:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772273.074746:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85400. 00010000:00000010:0.1:1775772273.200634:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772273.229923:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772273.239886:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772273.245514:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84c00. 00010000:00000010:0.1:1775772273.292030:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772273.310578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:2.1:1775772273.326856:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:2.1:1775772273.360315:0:2414:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:2.1:1775772273.380187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:2.1:1775772273.387842:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be5c00. 00010000:00000010:2.1:1775772273.417178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be5c00. 00010000:00000010:0.1:1775772273.547890:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:2.1:1775772273.555276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:2.1:1775772273.603308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772273.618182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772273.649427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772273.676412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772273.708193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772273.744594:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772273.813163:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:0.1:1775772273.914243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772273.973590:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772274.010712:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772274.036179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772274.053897:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be5e00. 00010000:00000010:0.1:1775772274.058205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be4400. 00010000:00000010:0.1:1775772274.078297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:2.1:1775772274.103762:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772274.141173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772274.220318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:2.1:1775772274.252196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00010000:00000010:2.1:1775772274.291788:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00010000:00000010:0.1:1775772274.339169:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772274.378190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:2.1:1775772274.398211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be4a00. 00010000:00000010:2.1:1775772274.431181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297600. 00010000:00000010:2.1:1775772274.484182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296600. 00010000:00000010:0.1:1775772274.511293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772274.526212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:2.1:1775772274.531147:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:2.1:1775772274.566216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:0.1:1775772274.635757:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772274.700462:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772274.705175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772274.744176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772274.762631:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772274.802371:0:207609:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772274.876175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:2.1:1775772274.891078:0:1358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85200. 00010000:00000010:0.1:1775772274.928440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:2.1:1775772274.947614:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772274.957183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772275.015237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:2.1:1775772275.046178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297e00. 00010000:00000010:2.1:1775772275.067172:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296600. 00010000:00000010:0.1:1775772275.074307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be5c00. 00010000:00000010:0.1:1775772275.102345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85200. 00010000:00000010:0.1:1775772275.156232:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772275.165175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772275.189163:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772275.219368:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772275.229393:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772275.265179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772275.303175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772275.312423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:2.1:1775772275.334187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772275.365695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772275.372331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772275.467186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772275.485193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772275.544198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85400. 00010000:00000010:0.1:1775772275.567049:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772275.647180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772275.673547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772275.777469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772275.813179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772275.887240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772275.913622:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772275.997181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772276.018574:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772276.089946:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772276.104277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772276.204522:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:2.1:1775772276.226331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772276.295194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772276.335712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772276.406386:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:0.1:1775772276.454333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00010000:00000010:2.1:1775772276.516180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84400. 00010000:00000010:0.1:1775772276.548702:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00010000:00000010:2.1:1775772276.607581:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297600. 00010000:00000010:2.1:1775772276.648170:0:2414:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296600. 00010000:00000010:2.1:1775772276.728177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296400. 00010000:00000010:0.1:1775772276.838439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297a00. 00010000:00000010:0.1:1775772276.859198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296e00. 00010000:00000010:0.1:1775772276.909220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772276.930811:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:0.1:1775772276.947247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:0.1:1775772276.973666:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772277.039189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772277.043189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772277.109192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772277.117525:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:2.1:1775772277.137195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772277.192192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772277.217179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772277.240658:0:1358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772277.310441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772277.328188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:2.1:1775772277.348182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:0.1:1775772277.370182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85400. 00010000:00000010:2.1:1775772277.402586:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85200. 00010000:00000010:0.1:1775772277.429740:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297600. 00010000:00000010:0.1:1775772277.448752:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772277.469809:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772277.508237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772277.521993:0:203109:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772277.546175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772277.589514:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297a00. 00010000:00000010:0.1:1775772277.661172:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772277.661184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772277.724052:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772277.785454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772277.861942:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85400. 00010000:00000010:0.1:1775772277.915579:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772277.941200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:0.1:1775772278.074413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772278.080257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772278.105726:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772278.203564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772278.214779:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:2.1:1775772278.257563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772278.287541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:2.1:1775772278.353849:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296e00. 00010000:00000010:2.1:1775772278.428191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296e00. 00010000:00000010:2.1:1775772278.480494:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00010000:00000010:2.1:1775772278.613416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:2.1:1775772278.621171:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772278.671198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:2.1:1775772278.738171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:2.1:1775772278.814154:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772278.928177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85200. 00010000:00000010:0.1:1775772278.928184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85400. 00010000:00000010:0.1:1775772278.942056:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772279.012182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772279.032451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772279.099375:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772279.132265:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:2.1:1775772279.140187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772279.194204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772279.219178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772279.262033:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772279.285523:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:2.1:1775772279.351960:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772279.370180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772279.406725:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:2.1:1775772279.428190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:0.1:1775772279.465189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772279.469203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:2.1:1775772279.518184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772279.570341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772279.613168:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772279.638823:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:0.1:1775772279.671195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772279.692258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:2.1:1775772279.717419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772279.740410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772279.770102:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772279.781799:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772279.813182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772279.828384:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772279.848266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772279.898188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772279.930276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772279.983245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00010000:00000010:2.1:1775772279.996025:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:0.1:1775772280.028354:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:0.1:1775772280.058487:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772280.089178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772280.126197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772280.135439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772280.160179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772280.198517:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772280.209634:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772280.289487:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772280.367156:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:2.1:1775772280.431733:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:2.1:1775772280.453187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772280.462185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:0.1:1775772280.482607:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be5600. 00010000:00000010:2.1:1775772280.496185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85200. 00010000:00000010:0.1:1775772280.536348:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296800. 00010000:00000010:2.1:1775772280.550178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be4a00. 00010000:00000010:0.1:1775772280.578322:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be4800. 00010000:00000010:0.1:1775772280.613244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be5c00. 00010000:00000010:0.1:1775772280.613255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be5800. 00010000:00000010:2.1:1775772280.618578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84c00. 00010000:00000010:2.1:1775772280.782170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772280.809324:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296600. 00010000:00000010:0.1:1775772280.858444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85200. 00010000:00000010:0.1:1775772280.873895:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772280.950831:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772280.975331:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772281.017168:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772281.017175:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772281.045700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:2.1:1775772281.045900:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772281.106633:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772281.120411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772281.151341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296400. 00010000:00000010:0.1:1775772281.192179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772281.214327:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772281.247174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85400. 00010000:00000010:0.1:1775772281.258181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772281.275171:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772281.298393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772281.371933:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297200. 00010000:00000010:0.1:1775772281.384183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772281.456348:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772281.563250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772281.572561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772281.581058:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:2.1:1775772281.600182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:0.1:1775772281.664431:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772281.700350:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772281.768489:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296a00. 00010000:00000010:0.1:1775772281.811188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772281.896092:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772282.009022:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:2.1:1775772282.061459:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772282.093187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772282.129216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772282.173758:0:48:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772282.210978:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772282.271360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:2.1:1775772282.299287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772282.304123:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:2.1:1775772282.340494:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:2.1:1775772282.340501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772282.343330:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296400. 00010000:00000010:2.1:1775772282.368214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296a00. 00010000:00000010:2.1:1775772282.401186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297600. 00010000:00000010:2.1:1775772282.430380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297000. 00010000:00000010:0.1:1775772282.502431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772282.512171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772282.586179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772282.586200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:2.1:1775772282.616200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297000. 00010000:00000010:0.1:1775772282.632748:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297600. 00010000:00000010:0.1:1775772282.673173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772282.703772:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772282.742116:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84400. 00010000:00000010:0.1:1775772282.770180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772282.815004:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297c00. 00010000:00000010:0.1:1775772282.863163:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772282.884389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772282.918084:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:2.1:1775772282.926675:0:2414:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772282.938201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00010000:00000010:0.1:1775772282.966944:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:0.1:1775772282.977171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be4600. 00010000:00000010:2.1:1775772282.981985:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:0.1:1775772283.024830:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772283.024844:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772283.024847:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be4000. 00010000:00000010:2.1:1775772283.048276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:2.1:1775772283.113198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772283.162905:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772283.176171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772283.249172:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772283.309438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772283.313408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772283.339279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772283.384558:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772283.437120:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772283.447358:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772283.481308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772283.490881:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772283.571326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772283.712353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772283.748013:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:2.1:1775772283.765368:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772283.792704:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772283.813323:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772283.834649:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772283.840263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772283.849286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:0.1:1775772283.880711:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296a00. 00010000:00000010:0.1:1775772283.904964:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772283.931197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84400. 00010000:00000010:0.1:1775772283.949505:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00010000:00000010:2.1:1775772284.038219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297a00. 00010000:00000010:0.1:1775772284.106384:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772284.143174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772284.186192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772284.234436:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772284.281419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772284.311459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772284.349195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772284.384660:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772284.391201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772284.436294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:2.1:1775772284.450933:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:2.1:1775772284.518186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296a00. 00010000:00000010:2.1:1775772284.528574:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296c00. 00010000:00000010:2.1:1775772284.554191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296a00. 00010000:00000010:0.1:1775772284.601422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be4000. 00010000:00000010:0.1:1775772284.623708:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296400. 00010000:00000010:0.1:1775772284.652516:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:2.1:1775772284.669368:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772284.669431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772284.705515:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be4c00. 00010000:00000010:0.1:1775772284.729278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772284.754178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772284.759346:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772284.774193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772284.785305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772284.814174:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772284.829265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772284.846700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:0.1:1775772284.867301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85200. 00010000:00000010:0.1:1775772284.883347:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772284.910197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772284.945421:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772284.970311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84c00. 00010000:00000010:0.1:1775772285.004339:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85400. 00010000:00000010:0.1:1775772285.019549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772285.024342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772285.038326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772285.089537:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772285.102963:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:2.1:1775772285.136196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:2.1:1775772285.157295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:2.1:1775772285.204189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00010000:00000010:0.1:1775772285.227275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:0.1:1775772285.252023:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:2.1:1775772285.259504:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772285.479693:0:203109:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296a00. 00010000:00000010:0.1:1775772285.479700:0:203109:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296800. 00010000:00000010:0.1:1775772285.500693:0:203108:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772285.527725:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772285.530373:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772285.534579:0:207609:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772285.561834:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772285.591247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772285.608274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772285.652181:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772285.659334:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772285.717942:0:207610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772285.726523:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772285.793318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772285.810326:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772285.820946:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772285.837505:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772285.841596:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772285.857313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772285.882186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:2.1:1775772285.882313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772285.933373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772285.967424:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772285.981679:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772286.030022:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772286.082174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85400. 00010000:00000010:0.1:1775772286.120113:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:0.1:1775772286.149620:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772286.187616:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772286.194417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772286.237317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772286.312165:0:207609:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772286.322625:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:2.1:1775772286.345584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:0.1:1775772286.400272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84400. 00010000:00000010:0.1:1775772286.407566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772286.407571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772286.457183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772286.464384:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:2.1:1775772286.496293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:0.1:1775772286.508216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:2.1:1775772286.579305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85200. 00010000:00000010:0.1:1775772286.624257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772286.696250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772286.716178:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:2.1:1775772286.729185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772286.787529:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772286.939047:0:207609:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296a00. 00010000:00000010:0.1:1775772286.949269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be5600. 00010000:00000010:2.1:1775772286.963039:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be5000. 00010000:00000010:0.1:1775772287.012180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772287.096910:0:203109:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772287.109586:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:2.1:1775772287.155411:0:2415:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772287.177597:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772287.181371:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00010000:00000010:0.1:1775772287.251288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772287.329094:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84400. 00010000:00000010:2.1:1775772287.386247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772287.456174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772287.532297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772287.608258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772287.693446:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772287.698298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84400. 00010000:00000010:0.1:1775772287.749186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772287.826897:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772287.957624:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00010000:00000010:0.1:1775772287.957638:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:0.1:1775772288.012207:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:0.1:1775772288.078292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772288.089872:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772288.128220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772288.187501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772288.251244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:2.1:1775772288.283176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772288.308180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:2.1:1775772288.348891:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772288.436215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:2.1:1775772288.469483:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:0.1:1775772288.493823:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:0.1:1775772288.554493:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772288.612799:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297800. 00010000:00000010:2.1:1775772288.712192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296400. 00010000:00000010:0.1:1775772288.744250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:2.1:1775772288.765302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772288.840185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772288.950866:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772289.006344:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772289.086611:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772289.140292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:2.1:1775772289.153591:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772289.184275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:2.1:1775772289.247170:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772289.298199:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:2.1:1775772289.331920:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772289.408252:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772289.445297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772289.456231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772289.487841:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772289.500263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:2.1:1775772289.535192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772289.590386:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84400. 00010000:00000010:0.1:1775772289.629898:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772289.660297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772289.710196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772289.736268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772289.799114:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772289.834297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772289.883390:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:0.1:1775772289.921271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772289.930654:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772289.949246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772289.962197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772289.973282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772290.009388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772290.040644:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772290.063409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772290.105748:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:0.1:1775772290.133387:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772290.176206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772290.204193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84c00. 00010000:00000010:0.1:1775772290.267263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772290.287204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:2.1:1775772290.319372:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:2.1:1775772290.319385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772290.333831:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772290.343240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772290.380741:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772290.410197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772290.418243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85200. 00010000:00000010:0.1:1775772290.428199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85400. 00010000:00000010:0.1:1775772290.463283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84c00. 00010000:00000010:0.1:1775772290.495347:0:1029:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772290.555506:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772290.632319:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772290.724551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772290.776358:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772290.834181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772290.968191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772291.027190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772291.047487:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:2.1:1775772291.047997:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772291.073277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772291.110871:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772291.118236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772291.135333:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:2.1:1775772291.149505:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772291.159348:0:203108:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772291.180197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772291.212664:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772291.258614:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:0.1:1775772291.325169:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772291.378316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772291.460205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772291.471169:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772291.483542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772291.516544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772291.527330:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772291.537176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:0.1:1775772291.560259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772291.570482:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772291.587187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772291.629175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:2.1:1775772291.629223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772291.643621:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:2.1:1775772291.661376:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772291.665240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772291.675699:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772291.708591:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772291.740612:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772291.770182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772291.786172:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772291.792298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772291.825179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772291.838216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:2.1:1775772291.851939:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772291.863243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772291.887280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772291.916395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772291.952228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00010000:00000010:0.1:1775772291.966821:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772291.987194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:2.1:1775772292.010284:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772292.021336:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772292.066701:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84400. 00010000:00000010:0.1:1775772292.123411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772292.193311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772292.250183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772292.299364:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:0.1:1775772292.329516:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:2.1:1775772292.371350:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:0.1:1775772292.385236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772292.435182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:2.1:1775772292.450242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:2.1:1775772292.490189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85400. 00010000:00000010:0.1:1775772292.561745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84c00. 00010000:00000010:0.1:1775772292.609339:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772292.666372:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:2.1:1775772292.668373:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85400. 00010000:00000010:0.1:1775772292.758188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:2.1:1775772292.807246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:0.1:1775772292.854171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00010000:00000010:0.1:1775772292.883181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772292.886570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:2.1:1775772292.977871:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772292.991415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84400. 00010000:00000010:2.1:1775772293.010438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00010000:00000010:0.1:1775772293.085997:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:2.1:1775772293.087750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be5000. 00010000:00000010:0.1:1775772293.111312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296a00. 00010000:00000010:0.1:1775772293.134548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772293.152178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772293.163185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772293.176190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772293.189186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772293.239258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772293.244176:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772293.277031:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772293.310368:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772293.315254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772293.346385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:0.1:1775772293.369261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:0.1:1775772293.384335:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:0.1:1775772293.458312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772293.475586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234be5e00. 00010000:00000010:0.1:1775772293.485026:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84c00. 00010000:00000010:0.1:1775772293.520709:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772293.527111:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772293.623790:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772293.710031:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772293.714534:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:2.1:1775772293.725199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772293.774343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772293.787331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772293.826297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772293.853724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:0.1:1775772293.865247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:2.1:1775772293.895179:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84400. 00010000:00000010:0.1:1775772293.900570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:0.1:1775772293.913198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772293.957168:0:1029:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775772293.963223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772293.979181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772294.011199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:2.1:1775772294.022215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:0.1:1775772294.032247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:0.1:1775772294.043485:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85400. 00010000:00000010:0.1:1775772294.079155:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772294.106198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772294.147697:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772294.187226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84c00. 00010000:00000010:0.1:1775772294.191274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772294.217266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:0.1:1775772294.268729:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772294.276695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:0.1:1775772294.289561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:0.1:1775772294.335002:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772294.335026:0:2413:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772294.368958:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772294.401664:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:2.1:1775772294.437316:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772294.454324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:2.1:1775772294.469166:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772294.487884:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772294.512324:0:203109:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772294.530798:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772294.551736:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00010000:00000010:0.1:1775772294.575183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297a00. 00010000:00000010:0.1:1775772294.579203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f84800. 00010000:00000010:0.1:1775772294.599197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:0.1:1775772294.623853:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772294.636703:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772294.663275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772294.706478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775772294.760189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772294.788592:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:2.1:1775772294.839234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00010000:00000010:0.1:1775772294.854333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772294.956415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00010000:00000010:0.1:1775772295.009930:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85c00. 00010000:00000010:2.1:1775772295.055726:0:208387:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775772295.056940:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85800. 00010000:00000010:0.1:1775772295.107273:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9e00. 00010000:00000010:0.1:1775772295.184903:0:2412:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775772295.240367:0:2404:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00010000:00000010:0.1:1775772295.290189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:2.1:1775772329.129174:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990600. 00010000:00000010:2.1:1775772329.129182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6212f85e00. 00010000:00000010:0.1:1775772341.329375:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3700. 00010000:00000010:0.1:1775772341.329389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8000. 00010000:00000010:0.1:1775772341.366069:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0200. 00010000:00000010:0.1:1775772341.723095:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3800. 00010000:00000010:0.1:1775772341.723103:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9a00. 00010000:00000010:0.1:1775772341.725345:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0900. 00010000:00000010:0.1:1775772341.729416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297600. 00010000:00000010:0.1:1775772784.575257:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6212fbbc00. 00010000:00000010:0.1:1775772784.575265:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e40ca00. 00010000:00000010:0.1:1775772784.575268:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6212fbb100. 00010000:00000010:0.1:1775772784.575270:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e40c000. 00010000:00000010:0.1:1775772784.575274:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c25fd00. 00010000:00000010:0.1:1775772784.575277:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101153000. 00010000:00000010:0.1:1775772784.575281:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c25f200. 00010000:00000010:0.1:1775772784.575284:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101123e00. 00010000:00000010:0.1:1775772784.575443:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c25f400. 00010000:00000010:0.1:1775772784.575447:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101123200. 00010000:00000010:0.1:1775772784.575449:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6212fbb600. 00010000:00000010:0.1:1775772784.575525:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e40cc00. 00010000:00000010:0.1:1775772784.575527:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6212fbb200. 00010000:00000010:0.1:1775772784.575530:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e40c200. 00010000:00000010:0.1:1775772784.575532:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6212fbb700. 00010000:00000010:0.1:1775772784.575535:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167f200. 00010000:00000010:0.1:1775772784.575540:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae2500. 00010000:00000010:0.1:1775772784.575544:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335400. 00010000:00000010:0.1:1775772784.575547:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e334c00. 00010000:00000010:0.1:1775772784.575550:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae2d00. 00010000:00000010:0.1:1775772784.575552:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335c00. 00010000:00000010:0.1:1775772784.575554:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c25f700. 00010000:00000010:0.1:1775772784.575557:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101122400. 00010000:00000010:0.1:1775772784.575569:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae2900. 00010000:00000010:0.1:1775772784.575573:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7200. 00010000:00000010:0.1:1775772784.575577:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6c00. 00010000:00000010:0.1:1775772784.575581:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5d00. 00010000:00000010:0.1:1775772784.575583:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d6400. 00010000:00000010:0.1:1775772784.575585:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae2a00. 00010000:00000010:0.1:1775772784.575587:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e334200. 00010000:00000010:0.1:1775772784.575589:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62055c5100. 00010000:00000010:0.1:1775772784.575591:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5d7400. 00010000:00000010:0.1:1775772784.575595:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226004600. 00010000:00000010:0.1:1775772784.575598:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6c00. 00010000:00000010:0.1:1775772784.575601:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226004000. 00010000:00000010:0.1:1775772784.575604:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101126200. 00010000:00000010:0.1:1775772784.575607:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6500. 00010000:00000010:0.1:1775772784.575610:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e334e00. 00010000:00000010:0.1:1775772784.575612:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62339d6f00. 00010000:00000010:0.1:1775772784.575614:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101126600. 00010000:00000010:0.1:1775772784.575616:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae2f00. 00010000:00000010:0.1:1775772784.575619:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335200. 00010000:00000010:0.1:1775772805.231228:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775772805.231237:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3d00. 00010000:00000010:0.1:1775772805.231241:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205297e00. 00010000:00000010:0.1:1775772805.231244:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990300. 00010000:00000010:0.1:1775772805.231247:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206b3da00. 00010000:00000010:0.1:1775772805.231250:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88a00. 00010000:00000010:0.1:1775772805.231253:0:209912:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc64400. 00010000:00000010:0.1:1775772805.231268:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc48a00. 00010000:00000010:0.1:1775772805.231272:0:209912:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88c00. 00000800:00000200:0.1:1775772876.166115:0:2405:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 733028 low water 1730638 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775772876.166183:0:2405:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 733028 low water 1730638 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775772984.868469:0:2405:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775772984.868487:0:2405:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775772984.868498:0:2405:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775772984.868511:0:2405:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00010000:00000010:2.1:1775773020.396244:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0300. 00010000:00000010:2.1:1775773020.396253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296600. 00010000:00000010:2.1:1775773022.389178:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0900. 00010000:00000010:2.1:1775773022.389189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205296800. 00010000:00000010:2.1:1775773022.389202:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990900. 00010000:00000010:2.1:1775773022.389206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101152a00. 00010000:00000010:0.1:1775773022.441524:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775773022.441541:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0400. 00010000:00000010:0.1:1775773022.441545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206b3ce00. 00010000:00000010:0.1:1775773022.441563:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3900. 00010000:00000010:0.1:1775773022.441567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00000080:00000010:0.1:1775773022.464191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057ebca0. 00000080:00000010:0.1:1775773022.687942:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f62192da100. 00010000:00000010:2.1:1775773022.707452:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb100. 00010000:00000010:2.1:1775773022.707468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88e00. 00000080:00000010:2.1:1775773022.764214:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62e80. 00010000:00000010:0.1:1775773022.793206:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0b00. 00010000:00000010:0.1:1775773022.793221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345eda00. 00010000:00000010:0.1:1775773022.793235:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0e00. 00010000:00000010:0.1:1775773022.801445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345ed000. 00010000:00000010:2.1:1775773022.967280:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202ae7600. 00010000:00000010:2.1:1775773022.967294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e334800. 00000800:00000200:2.1:1775773023.979188:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -35852 low water 2115078 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:2.1:1775773023.979204:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -35852 low water 2115078 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:2.1:1775773023.979216:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -35852 low water 2115078 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:2.1:1775773023.979227:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -35852 low water 2115078 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773026.627536:0:181953:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773026.627548:0:181953:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773026.627558:0:181953:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773026.627567:0:181953:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773026.627576:0:181953:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00010000:00000010:0.1:1775773027.530253:0:210021:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3400. 00010000:00000010:0.1:1775773027.530264:0:210021:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8800. 00010000:00000010:0.1:1775773027.571926:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3c00. 00010000:00000010:0.1:1775773027.571934:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8a00. 00000800:00000200:0.1:1775773030.971849:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -49476 low water 2121890 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773030.971859:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -49476 low water 2121890 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773030.971868:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -49476 low water 2121890 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773030.971877:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -49476 low water 2121890 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773030.971885:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -49476 low water 2121890 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773032.909876:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773032.909888:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773032.909897:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773032.909906:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773032.909915:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773043.269293:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1255096 low water 1469604 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773046.631855:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773046.631869:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773046.631878:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773046.631888:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773046.631896:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773054.207868:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773054.208044:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773054.208341:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773054.208494:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773054.208653:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773054.208813:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773054.208969:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773054.209114:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773054.209262:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773054.209278:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773054.209289:0:2404:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.663039:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.663048:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.663055:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.663064:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.666624:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.666637:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.666647:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.666655:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.666664:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.666669:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.666706:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.667821:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.667842:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.667851:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.667858:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.667866:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.668352:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.668363:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.668371:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.668379:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.668387:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.669038:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.669050:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.669059:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.669067:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.669075:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.669643:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.669657:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.669666:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.669675:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.669683:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.670593:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.670606:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.670617:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.670626:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.670636:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.673165:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.673182:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.673194:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.673204:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.673215:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.673221:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.674679:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 2212908 low water 990698 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.823231:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.823249:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.823260:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.823271:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.823283:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.825736:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.825773:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.825788:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.825798:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.825808:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.825818:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.825824:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.828625:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.828808:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.828833:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.828847:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.828857:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.828866:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.828875:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.829203:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.829228:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.832805:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.833017:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.833045:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.833055:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.833064:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.833072:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.833080:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.837229:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.837419:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.837971:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.838284:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.838554:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.838727:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.838907:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.839106:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.839379:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.839578:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.839748:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.839924:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.840117:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.840967:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.840998:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.841010:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.841020:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.842661:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.842698:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.842708:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.842717:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.842727:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.843216:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935004 low water 1629650 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773063.843263:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1573748 low water 1310278 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.252572:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.252591:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.252604:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.252615:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.252625:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.253040:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.253056:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.253066:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.253077:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.253087:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.254370:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.254384:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.254392:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.254400:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.254408:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.254800:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.254813:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.254822:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.254830:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.254838:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.255235:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.255249:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.255260:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.255270:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.255280:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.255709:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.255724:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.255734:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.255744:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.255752:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.259930:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935004 low water 1629650 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.259953:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935004 low water 1629650 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.259964:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935004 low water 1629650 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.259976:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935004 low water 1629650 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.259987:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935004 low water 1629650 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.259993:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935004 low water 1629650 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.260052:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.263122:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.263136:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.263146:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.263169:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.263179:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.271152:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.271168:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.271177:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.271187:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.271196:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.271201:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.271257:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1957244 low water 1118530 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.641081:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.641101:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.641111:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.641120:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.641130:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.641790:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.641808:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.641817:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.641825:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.641834:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.642319:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.642331:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.642340:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.642348:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.642356:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.643073:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.643084:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.643092:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.643100:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.643109:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.644431:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.644449:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.644471:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.644480:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.644488:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.646747:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.646760:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.646768:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.646777:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.646785:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.650462:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.650486:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.650497:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.650506:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.650517:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.650523:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.651174:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.651190:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.651201:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.651213:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.651224:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.652226:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.652243:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.652252:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.652261:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.652269:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.653846:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615424 low water 1789440 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.653865:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615424 low water 1789440 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.653875:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615424 low water 1789440 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.653885:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615424 low water 1789440 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.653894:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615424 low water 1789440 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.653901:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615424 low water 1789440 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773064.653958:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1573748 low water 1310278 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.023595:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.023628:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.023640:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.023650:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.023661:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.025209:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.025229:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.025241:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.025252:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.025264:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.025702:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.025715:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.025724:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.025732:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.025740:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.026223:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.026238:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.026248:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.026258:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.026266:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.026708:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.026723:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.026733:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.026744:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.026754:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36940 low water 2115622 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.028627:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551092 low water 1821606 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.028646:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551092 low water 1821606 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.028655:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551092 low water 1821606 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.028665:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551092 low water 1821606 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.028675:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551092 low water 1821606 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.028681:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551092 low water 1821606 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.029125:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551092 low water 1821606 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.029139:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551092 low water 1821606 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.029148:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551092 low water 1821606 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.029156:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551092 low water 1821606 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.029164:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551092 low water 1821606 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.030494:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551092 low water 1821606 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.030515:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551092 low water 1821606 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.030526:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551092 low water 1821606 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.030537:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551092 low water 1821606 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.030547:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551092 low water 1821606 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.041776:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 2532488 low water 830908 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.077996:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -30636 low water 2112470 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.078013:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -30636 low water 2112470 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.078024:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -30636 low water 2112470 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.078033:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -30636 low water 2112470 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.078043:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -30636 low water 2112470 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.078482:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -30636 low water 2112470 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.078496:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -30636 low water 2112470 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.078505:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -30636 low water 2112470 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.078514:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -30636 low water 2112470 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.078523:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -30636 low water 2112470 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.078958:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -30636 low water 2112470 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.078969:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -30636 low water 2112470 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.078978:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -30636 low water 2112470 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.078986:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -30636 low water 2112470 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.078994:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -30636 low water 2112470 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.079430:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -30636 low water 2112470 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.079445:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -30636 low water 2112470 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.079456:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -30636 low water 2112470 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.079468:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -30636 low water 2112470 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.079479:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -30636 low water 2112470 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.081861:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615080 low water 1789612 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.081881:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615080 low water 1789612 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.081891:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615080 low water 1789612 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.081902:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615080 low water 1789612 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.081911:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615080 low water 1789612 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.081916:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615080 low water 1789612 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.082877:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615080 low water 1789612 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.082889:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615080 low water 1789612 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.082898:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615080 low water 1789612 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.082906:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615080 low water 1789612 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.082913:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615080 low water 1789612 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.085134:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 678996 low water 1757654 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.087558:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 678996 low water 1757654 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.087612:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 678996 low water 1757654 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.087627:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 678996 low water 1757654 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.087638:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 678996 low water 1757654 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.087648:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 678996 low water 1757654 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.087658:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 678996 low water 1757654 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.087674:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 678996 low water 1757654 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.089735:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1509832 low water 1342236 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.151130:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.151350:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.151527:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.151692:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.151926:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.152820:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.153076:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.153346:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.153528:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.153692:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.154743:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.154772:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.154783:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.154792:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.154800:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.154809:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.154823:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.159541:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.159822:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.159851:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.159866:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.159877:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.159888:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.159898:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.160888:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 870672 low water 1661816 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.160905:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 870672 low water 1661816 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.160916:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 870672 low water 1661816 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.160926:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 870672 low water 1661816 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.160936:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 870672 low water 1661816 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.160942:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 870672 low water 1661816 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.160983:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1509832 low water 1342236 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.235589:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.235601:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.235610:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.235618:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.235626:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.237710:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.237733:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.237744:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.237757:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.237766:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.241864:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.241881:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.246185:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 2724236 low water 735034 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.489178:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.489199:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.489211:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.489222:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.489233:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.490299:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.490318:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.490330:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.490341:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.490351:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.490981:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.490995:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.491014:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.491024:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.491035:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.491041:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.491059:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.492398:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.492426:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.492438:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.492448:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.492458:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.492465:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.493101:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.493115:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.493126:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.493137:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.493147:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.493153:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.494328:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.494345:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.494357:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.494368:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.494378:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.495373:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.495387:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.495411:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.495424:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.495434:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.495442:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.495550:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1829412 low water 1182446 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.798498:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -16640 low water 2105472 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.798514:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -16640 low water 2105472 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.798526:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -16640 low water 2105472 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.798536:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -16640 low water 2105472 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.798545:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -16640 low water 2105472 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.800626:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -16640 low water 2105472 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.800642:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -16640 low water 2105472 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.800653:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -16640 low water 2105472 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773065.814720:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1637664 low water 1278320 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.268502:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.268523:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.268540:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.268559:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.268573:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.269414:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.269444:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.269461:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.269841:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.270047:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.270305:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.270591:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.270775:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.270967:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.270998:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.271011:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.271021:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.271032:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.271043:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.272828:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.272850:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.272866:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.272881:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.272897:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.276642:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.276664:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.276679:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.276695:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.276708:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.277198:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.278878:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.278897:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.278909:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.278919:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.278927:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.279054:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.281746:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.281769:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.281786:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.281801:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.281817:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.285548:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.285571:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.285588:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.285605:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.285622:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.287818:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.287839:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.287854:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.287869:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.287883:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.289006:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.289026:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.289042:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.289059:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.289075:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.289085:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.289116:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1318084 low water 1438110 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.289582:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 2276824 low water 958740 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.328889:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.328908:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.328918:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.328926:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.328935:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.333887:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.333907:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.333916:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.333924:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.333933:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.333939:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.334348:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.334363:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.334372:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.334381:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.334405:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.335698:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.335735:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.335749:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.335760:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.335771:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.335782:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.336309:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.336717:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.336749:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.336762:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.336772:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.336782:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.336790:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.340096:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.340113:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.340123:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.340130:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.340138:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.340740:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.340753:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.340761:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.340770:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.340778:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.340782:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.340812:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.342041:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.342055:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.342063:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.342071:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.342079:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.342849:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.342862:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.342872:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.342880:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.342888:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.343717:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.343729:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.343739:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.343748:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.343757:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.345053:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.345074:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.345084:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.345092:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.345100:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.345108:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.349788:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1829412 low water 1182446 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.483641:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.483654:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.483664:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.483672:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.483680:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.483685:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.483997:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.484007:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.484017:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.484025:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.484033:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.487226:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.487239:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.487248:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.487257:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.487265:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.487847:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.490105:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.490477:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.490758:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.491037:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.491506:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.491779:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.491973:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.491987:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.491996:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.492005:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.492014:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.492024:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.492834:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.492845:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.492853:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.492861:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.492869:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.493721:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.493733:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.493741:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.493748:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.493756:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.494976:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.494994:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.495003:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.495011:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.495019:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.495598:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.495611:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.495620:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.495629:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.495639:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.497284:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.497301:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.497310:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.497319:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.497330:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.497731:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.497745:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.497753:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.497761:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.497770:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.498260:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.498277:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.498288:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.498298:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.501841:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 2852068 low water 671118 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.702895:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295428 low water 1949438 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.705162:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295428 low water 1949438 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.705472:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295428 low water 1949438 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.705810:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295428 low water 1949438 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.706295:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295428 low water 1949438 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.707433:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295428 low water 1949438 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.709095:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295428 low water 1949438 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.709171:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295428 low water 1949438 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.709187:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295428 low water 1949438 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.709200:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295428 low water 1949438 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.709213:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295428 low water 1949438 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.709224:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295428 low water 1949438 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.709239:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295428 low water 1949438 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.709294:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 423260 low water 1885522 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.709318:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 678924 low water 1757690 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.713666:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 742840 low water 1725732 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.713680:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 742840 low water 1725732 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.713690:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 742840 low water 1725732 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.713700:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 742840 low water 1725732 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.713710:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 742840 low water 1725732 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.713716:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 742840 low water 1725732 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.713744:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1062420 low water 1565942 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.721834:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1126336 low water 1533984 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.722037:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1126336 low water 1533984 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.726753:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1126336 low water 1533984 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.726833:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1126336 low water 1533984 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.726847:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1126336 low water 1533984 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.726856:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1126336 low water 1533984 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.726865:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1126336 low water 1533984 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.726873:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1126336 low water 1533984 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.726888:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1126336 low water 1533984 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.726948:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.727903:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.727915:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.727924:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.727932:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.727941:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.728294:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.728306:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.728315:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.728325:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.728334:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.730927:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.730940:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.730949:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.730957:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.730965:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.732250:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 2212908 low water 990698 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.798941:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.798954:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.798963:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.798971:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.798986:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.813283:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.813296:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.813306:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.813314:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.813323:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.813328:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.813343:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.813386:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.813439:0:2414:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1829412 low water 1182446 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.972908:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.973117:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.973330:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.973512:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.973691:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.973955:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.974042:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.974059:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.974072:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.974083:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.974095:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.976723:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.980511:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.980561:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.980574:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.980586:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.980596:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.981502:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.981517:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.981525:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.981534:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.981542:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.981548:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.981588:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.981622:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935004 low water 1629650 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.981638:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.985434:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.985454:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.985466:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.985476:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.985487:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.990673:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.990696:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.990707:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.990717:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.990726:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.996236:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.996252:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.996262:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.996273:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.996282:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.999806:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.999829:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.999838:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.999848:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773066.999856:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.000577:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1509832 low water 1342236 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.206399:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.206417:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.206428:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.206438:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.206448:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.206453:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.211340:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.211356:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.211367:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.211393:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.211403:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.211410:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.211436:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.211856:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.211871:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.211881:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.211890:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.211897:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.212829:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.212843:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.212853:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.212863:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.212874:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.216865:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.216882:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.216894:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.216905:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.216916:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.218471:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.218491:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.218502:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.218512:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.218522:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.221674:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.221688:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.221697:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.221706:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.221714:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.221720:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.221759:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.223959:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.223972:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.223992:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.224001:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.224009:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.224015:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.224056:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1829412 low water 1182446 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.265884:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 456032 low water 1869136 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.265901:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 456032 low water 1869136 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.265912:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 456032 low water 1869136 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.265923:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 456032 low water 1869136 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.265934:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 456032 low water 1869136 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.265939:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 456032 low water 1869136 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.265961:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 519948 low water 1837178 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.266022:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.267928:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.268322:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.268342:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.268360:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.268388:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.268403:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.270793:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1445916 low water 1374194 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.448582:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.448598:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.448610:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.448620:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.448631:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.448637:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.453440:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.453461:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.453473:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.453483:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.453494:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.457556:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.457575:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.457586:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.457596:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.457604:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.461325:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.461342:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.461352:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.461363:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.461387:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.461394:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.461439:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.462511:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.462526:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.462535:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.462544:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.462554:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.462559:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.463736:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.463755:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.463765:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.463774:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.463782:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.464431:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.464449:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.464463:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.464480:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.464495:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.465188:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.465205:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.465223:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.465239:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.465255:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.465973:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.466001:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.466015:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.466030:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.466046:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.466778:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.466795:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.466812:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.466823:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.466837:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.467597:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.467610:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.467619:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.467628:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.467637:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.468196:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.468208:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.470935:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.471007:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1829412 low water 1182446 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.578864:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 168496 low water 2012904 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.579598:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 168496 low water 2012904 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.580333:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 168496 low water 2012904 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.580374:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 168496 low water 2012904 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.580391:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 168496 low water 2012904 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.584584:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 168496 low water 2012904 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.584602:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 168496 low water 2012904 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.584613:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 168496 low water 2012904 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.584622:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 168496 low water 2012904 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.584631:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 168496 low water 2012904 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.584645:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 168496 low water 2012904 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.584696:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 232412 low water 1980946 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.584713:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 296328 low water 1948988 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.584728:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 360244 low water 1917030 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.584791:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 488076 low water 1853114 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.591574:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615908 low water 1789198 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.591593:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615908 low water 1789198 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.591604:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615908 low water 1789198 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.591614:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615908 low water 1789198 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.591622:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615908 low water 1789198 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.591628:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615908 low water 1789198 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.591655:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.592926:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.592944:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.592966:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.592983:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.592995:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.595620:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.595656:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.595673:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.595686:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.595701:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.595717:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.598988:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.599005:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.599014:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.599022:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.599030:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.601014:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.601032:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.601043:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.601053:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.601064:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.602887:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.602922:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.602939:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.602955:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.602979:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.602993:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.609569:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.609584:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.609596:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.609607:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.609617:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.610822:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.610839:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.610850:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.610859:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.610867:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.612657:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.613047:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.613067:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.613076:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.613083:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.613091:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.613098:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.614247:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935488 low water 1629408 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.617722:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 3746892 low water 223706 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.646582:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.646599:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.646611:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.646621:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.646629:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.647434:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.647450:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.647458:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.647891:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.647909:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.647918:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.647926:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.647936:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.647943:0:195265:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.649610:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.649626:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.649636:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.649646:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.649655:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.650684:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.650845:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.651601:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.651633:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.651644:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.651653:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.651662:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.651670:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.653156:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.653171:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.653181:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.653190:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.653199:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.656429:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807172 low water 1693566 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.656453:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 805892 low water 1694206 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.660133:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 869808 low water 1662248 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.660260:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 2531208 low water 831548 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773067.719187:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1509832 low water 1342236 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.069218:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -26112 low water 2110208 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.069233:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -26112 low water 2110208 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.069243:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -26112 low water 2110208 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.069251:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -26112 low water 2110208 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.069260:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -26112 low water 2110208 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.073671:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 923148 low water 1635578 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.073686:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 923148 low water 1635578 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.073695:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 923148 low water 1635578 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.073703:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 923148 low water 1635578 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.073713:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 923148 low water 1635578 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.073717:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 923148 low water 1635578 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.077392:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 991244 low water 1601530 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.077715:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 991244 low water 1601530 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.077739:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 991244 low water 1601530 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.077749:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 991244 low water 1601530 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.077758:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 991244 low water 1601530 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.077766:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 991244 low water 1601530 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.077774:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 991244 low water 1601530 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.077780:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 991244 low water 1601530 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.077809:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1058144 low water 1568080 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.077838:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1701580 low water 1246362 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.345036:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.345049:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.345058:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.345066:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.345074:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.345078:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 231928 low water 1981188 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.346962:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.346979:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.346987:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.346995:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.347002:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.347007:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.347736:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.347748:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.347756:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.347765:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.347772:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.348141:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.348152:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.348160:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.348167:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.348175:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.349716:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.349731:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.349739:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.349747:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.349755:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.351532:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.351548:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.351556:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.351564:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.351571:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.351935:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.351953:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.351964:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.351974:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.351983:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.352366:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.352378:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.352386:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.352394:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.352402:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 295844 low water 1949230 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.356169:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 2468572 low water 862866 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.470154:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 423676 low water 1885314 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.470171:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 423676 low water 1885314 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.470184:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 423676 low water 1885314 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.470193:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 423676 low water 1885314 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.470204:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 423676 low water 1885314 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.470211:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 423676 low water 1885314 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.470702:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 423676 low water 1885314 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.470718:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 423676 low water 1885314 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.470728:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 423676 low water 1885314 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.470737:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 423676 low water 1885314 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.470746:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 423676 low water 1885314 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.471235:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 423676 low water 1885314 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.471250:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 423676 low water 1885314 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.471261:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 423676 low water 1885314 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.471270:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 423676 low water 1885314 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.471280:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 423676 low water 1885314 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.471791:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.471804:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.471813:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.471822:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.471830:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.471835:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.472290:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.472306:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.472318:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.472326:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.472334:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.472880:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.472892:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.472901:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.472912:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.472924:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.473412:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.473426:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.473437:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.473447:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.473456:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.474367:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.474384:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.474393:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.474403:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.474413:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487592 low water 1853356 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.475128:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:2.1:1775773068.475823:0:26:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:2.1:1775773068.475836:0:26:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:2.1:1775773068.475847:0:26:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:2.1:1775773068.475858:0:26:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.476682:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.476697:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.476708:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.476718:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.476730:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.479909:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.479927:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.479938:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.479956:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.479967:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.484060:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1445916 low water 1374194 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.723817:0:32:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.723830:0:32:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.723840:0:32:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.723849:0:32:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.723857:0:32:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.723862:0:32:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.723888:0:32:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935004 low water 1629650 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.723906:0:32:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.725420:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.725437:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.725445:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.725453:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.725462:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.725467:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.726490:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.726504:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.726515:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.726525:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.726535:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.727009:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.727021:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.727029:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.727037:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.727045:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.728081:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.728092:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.728101:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.728109:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.728117:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.728594:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.728608:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.728617:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.728626:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.728634:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.729271:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.729286:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.729296:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.729305:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.729315:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.730454:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.730467:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.730476:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.730483:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.730492:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.731399:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.731414:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.731423:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.731433:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.731441:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.732146:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.732160:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.733310:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1318084 low water 1438110 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.740355:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 2915984 low water 639160 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.823190:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551864 low water 1821220 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.824294:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551864 low water 1821220 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.825394:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551864 low water 1821220 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.825436:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551864 low water 1821220 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.825450:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551864 low water 1821220 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.825464:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551864 low water 1821220 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.825476:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551864 low water 1821220 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.825486:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551864 low water 1821220 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.825499:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551864 low water 1821220 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.832352:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615780 low water 1789262 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.832392:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 679696 low water 1757304 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.832404:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 743612 low water 1725346 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.832415:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 807528 low water 1693388 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.832498:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1062420 low water 1565942 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.832512:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1062420 low water 1565942 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.832522:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1062420 low water 1565942 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.832532:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1062420 low water 1565942 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.832543:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1062420 low water 1565942 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.832548:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1062420 low water 1565942 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.838589:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1126336 low water 1533984 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.838608:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1126336 low water 1533984 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.838617:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1126336 low water 1533984 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.838625:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1126336 low water 1533984 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.838633:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1126336 low water 1533984 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.838650:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1126336 low water 1533984 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.838674:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.839425:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.839440:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.839449:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.839459:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.839468:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.840260:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.840276:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.840288:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.840297:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.840306:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.841990:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.842010:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.842020:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.842030:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.842040:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.843654:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.843673:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.843683:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.843692:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.843701:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.849843:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.849860:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.849869:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.849879:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.849889:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.852047:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.852065:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.852077:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.852087:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.852095:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.852740:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.852754:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.852764:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.852776:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.852785:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.853324:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.853338:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.853361:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.853370:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1382000 low water 1406152 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773068.855415:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 2979900 low water 607202 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.214639:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.214661:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.214672:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.214682:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.214692:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.214697:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.214733:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773069.221543:0:2405:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773069.221560:0:2405:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773069.221571:0:2405:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773069.221581:0:2405:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:0.1:1775773069.221592:0:2405:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1190252 low water 1502026 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.229763:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.229781:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.229791:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.229801:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.229810:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.229816:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1254168 low water 1470068 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.229851:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1829412 low water 1182446 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.437581:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 267360 low water 1963472 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.437629:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 359344 low water 1917480 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.437703:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 678924 low water 1757690 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.437718:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 678924 low water 1757690 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.437731:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 678924 low water 1757690 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.437741:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 678924 low water 1757690 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.437752:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 678924 low water 1757690 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.437758:0:210021:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 678924 low water 1757690 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.440307:0:31:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.440323:0:31:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.441584:0:31:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.441602:0:31:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.441612:0:31:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.441620:0:31:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.441629:0:31:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.441641:0:31:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.446556:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1062420 low water 1565942 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.446571:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1062420 low water 1565942 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.446580:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1062420 low water 1565942 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.446590:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1062420 low water 1565942 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.446600:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1062420 low water 1565942 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.446606:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1062420 low water 1565942 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.446624:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1126336 low water 1533984 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.449305:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1445916 low water 1374194 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.451440:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.451457:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.451468:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.451477:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.451486:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -24152 low water 2109228 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.455492:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.455509:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.455520:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.455530:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.455539:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.455545:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.456574:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.456605:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.456616:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.456624:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.456634:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.456643:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 487176 low water 1853564 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.460097:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 934588 low water 1629858 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.460118:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 934588 low water 1629858 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.460130:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 934588 low water 1629858 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.460140:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 934588 low water 1629858 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.460151:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 934588 low water 1629858 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.460156:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 934588 low water 1629858 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.461523:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.461538:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.461550:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.461561:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.461571:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.461577:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.465591:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.465608:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.465616:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.465626:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.465635:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.467239:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.467256:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.467264:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.467271:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.467278:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.468680:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.468700:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.468711:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.468720:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.468730:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.470543:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.470564:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.470575:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.470587:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.470596:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.471202:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.471216:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.471225:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.471234:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.471244:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998504 low water 1597900 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.471840:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 2276824 low water 958740 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.479744:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -23140 low water 2108722 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.479757:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -23140 low water 2108722 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.479767:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -23140 low water 2108722 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.479776:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -23140 low water 2108722 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.479783:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -23140 low water 2108722 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.480385:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.480400:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.480411:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.480419:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.480426:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.480431:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.480921:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.480940:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.480949:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.480958:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.480967:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.481385:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.481398:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.481408:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.481417:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.481426:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.482167:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.482180:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.482189:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.482197:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.482205:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.482647:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.482659:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.482669:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.482677:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 616020 low water 1789142 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.489675:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 679936 low water 1757184 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.489699:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 679936 low water 1757184 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.489709:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 679936 low water 1757184 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.489718:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 679936 low water 1757184 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.489728:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 679936 low water 1757184 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.489734:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 679936 low water 1757184 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.489789:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1574760 low water 1309772 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.491629:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -23140 low water 2108722 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.491646:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -23140 low water 2108722 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.491657:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -23140 low water 2108722 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.491667:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -23140 low water 2108722 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.491676:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -23140 low water 2108722 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.492261:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -23140 low water 2108722 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.492276:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -23140 low water 2108722 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.492285:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -23140 low water 2108722 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.492296:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -23140 low water 2108722 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.492305:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -23140 low water 2108722 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.494273:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 40776 low water 2076764 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.494293:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 40776 low water 2076764 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.494303:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 40776 low water 2076764 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.494312:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 40776 low water 2076764 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.494323:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 40776 low water 2076764 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.494329:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 40776 low water 2076764 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.494369:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 360356 low water 1916974 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.494419:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 999516 low water 1597394 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.495206:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 999516 low water 1597394 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.495221:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 999516 low water 1597394 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.495232:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 999516 low water 1597394 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.495242:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 999516 low water 1597394 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.495251:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 999516 low water 1597394 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.496114:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 999516 low water 1597394 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.496131:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 999516 low water 1597394 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.496141:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 999516 low water 1597394 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.496150:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 999516 low water 1597394 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.496159:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 999516 low water 1597394 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.496555:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 999516 low water 1597394 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.496569:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 999516 low water 1597394 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.496578:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 999516 low water 1597394 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.496587:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 999516 low water 1597394 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.500239:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1063432 low water 1565436 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.500255:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1063432 low water 1565436 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.500266:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1063432 low water 1565436 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.500272:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1063432 low water 1565436 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.511001:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 2021160 low water 1086572 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.514288:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 2038432 low water 1077936 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.721872:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.721902:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.721913:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.721927:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.721937:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.721946:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.724806:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.724847:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.724860:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.724869:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.724877:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.724885:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.728039:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.728086:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.728100:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.728111:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.728121:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.728131:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.729792:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace -36524 low water 2115414 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.729889:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 551508 low water 1821398 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.729924:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 615424 low water 1789440 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.729947:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.731281:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.731298:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.731308:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.731317:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.731336:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 998920 low water 1597692 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.738061:0:2415:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1701580 low water 1246362 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.989534:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.989550:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.989560:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.990791:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.990825:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.990836:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.990845:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.990855:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.990866:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 871088 low water 1661608 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.993800:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935004 low water 1629650 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.993811:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935004 low water 1629650 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.993820:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935004 low water 1629650 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.993828:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935004 low water 1629650 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.993836:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935004 low water 1629650 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.993841:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 935004 low water 1629650 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.993866:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1318084 low water 1438110 conn ffff8f62079ae000 blocked idle queued 00000800:00000200:3.1:1775773069.993904:0:0:0:(socklnd_lib.c:612:ksocknal_write_space()) sk ffff8f620ac3bf00 wspace 1893328 low water 1150488 conn ffff8f62079ae000 blocked idle queued 00010000:00000010:2.1:1775773070.283177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206187800. 00010000:00000010:2.1:1775773070.283193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62345edc00. 00010000:00000010:2.1:1775773070.283198:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb300. 00010000:00000010:2.1:1775773070.283252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89c00. 00010000:00000010:2.1:1775773070.283257:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202ae7f00. 00010000:00000010:2.1:1775773070.283261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206186600. 00000080:00000010:2.1:1775773070.297643:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2240. 00000080:00000010:2.1:1775773070.306194:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff8f621875a940. 00010000:00000010:3.1:1775773070.319244:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcbd00. 00010000:00000010:3.1:1775773070.319257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89600. 00010000:00000010:3.1:1775773070.328189:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb700. 00010000:00000010:2.1:1775773070.338193:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcbd00. 00010000:00000010:2.1:1775773070.338209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89000. 00010000:00000010:3.1:1775773070.345192:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb500. 00010000:00000010:3.1:1775773070.345209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89600. 00000080:00000010:0.1:1775773070.372189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd620a0. 00010000:00000010:3.1:1775773070.383296:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202ae7b00. 00010000:00000010:3.1:1775773070.383307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206187000. 00010000:00000010:2.1:1775773070.399230:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb700. 00010000:00000010:2.1:1775773070.404186:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3200. 00010000:00000010:2.1:1775773070.404202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89800. 00010000:00000010:2.1:1775773070.417228:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3300. 00010000:00000010:2.1:1775773070.417243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9c00. 00000080:00000010:2.1:1775773070.455634:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62e80. 00010000:00000010:2.1:1775773070.459380:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3200. 00010000:00000010:2.1:1775773070.459396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8400. 00010000:00000010:2.1:1775773070.473367:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3c00. 00010000:00000010:2.1:1775773070.485188:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3200. 00010000:00000010:2.1:1775773070.485211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9200. 00010000:00000010:2.1:1775773070.499673:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3400. 00010000:00000010:2.1:1775773070.499689:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8400. 00000080:00000010:3.1:1775773070.544828:0:210037:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb9a0. 00010000:00000010:2.1:1775773070.560322:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcbe00. 00010000:00000010:2.1:1775773070.560336:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88800. 00010000:00000010:2.1:1775773070.584195:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3400. 00010000:00000010:2.1:1775773070.596188:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3200. 00010000:00000010:2.1:1775773070.596204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8400. 00010000:00000010:0.1:1775773070.601543:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3c00. 00010000:00000010:0.1:1775773070.601559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9200. 00000080:00000010:0.1:1775773070.629178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb9a0. 00010000:00000010:2.1:1775773075.434160:0:210331:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00000080:00000010:3.1:1775773075.465248:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb920. 00010000:00000010:3.1:1775773075.465271:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcbc00. 00010000:00000010:3.1:1775773075.465277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88400. 00010000:00000010:0.1:1775773075.564348:0:2405:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3b00. 00010000:00000010:0.1:1775773075.564368:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:2.1:1775773075.576593:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3800. 00010000:00000010:2.1:1775773075.576600:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:3.1:1775773075.800097:0:210337:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb400. 00010000:00000010:3.1:1775773075.800111:0:210337:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88c00. 00010000:00000010:3.1:1775773075.875381:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb400. 00010000:00000010:2.1:1775773076.049736:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8800. 00010000:00000010:3.1:1775773076.058915:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb400. 00000080:00000010:3.1:1775773076.081394:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb0a0. 00010000:00000010:3.1:1775773076.081417:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcba00. 00010000:00000010:3.1:1775773076.081422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88a00. 00010000:00000010:3.1:1775773076.215178:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcba00. 00000080:00000010:2.1:1775773076.226367:0:210346:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bea0. 00010000:00000010:2.1:1775773076.252555:0:210346:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8800. 00000080:00000010:3.1:1775773076.267800:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb4c0. 00000080:00000010:3.1:1775773076.275375:0:210347:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eba60. 00010000:00000010:2.1:1775773076.312226:0:210347:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8800. 00000080:00000010:3.1:1775773076.335533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb8a0. 00010000:00000010:3.1:1775773076.335545:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3800. 00010000:00000010:3.1:1775773076.335550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8200. 00010000:00000010:2.1:1775773076.395190:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3e00. 00000080:00000010:2.1:1775773076.409484:0:210349:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057ebc40. 00010000:00000010:2.1:1775773076.446189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8000. 00000080:00000010:2.1:1775773076.451663:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb500. 00000080:00000010:0.1:1775773076.458396:0:210350:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057ebac0. 00010000:00000010:2.1:1775773076.565184:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3c00. 00010000:00000010:2.1:1775773076.570370:0:210353:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206186200. 00000080:00000010:2.1:1775773076.570381:0:210353:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb860. 00010000:00000010:3.1:1775773076.597247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89800. 00000080:00000010:3.1:1775773076.605060:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62920. 00010000:00000010:2.1:1775773076.636166:0:210354:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9200. 00000080:00000010:2.1:1775773076.647195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb860. 00010000:00000010:2.1:1775773076.647225:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202ae7c00. 00010000:00000010:2.1:1775773076.647231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206187a00. 00010000:00000010:3.1:1775773076.676225:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb700. 00010000:00000010:3.1:1775773076.676234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89800. 00010000:00000010:3.1:1775773076.686199:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcbd00. 00000080:00000010:3.1:1775773076.686218:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62700. 00000080:00000010:2.1:1775773076.690281:0:210356:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62c60. 00010000:00000010:2.1:1775773076.708223:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3400. 00010000:00000010:2.1:1775773076.708257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89c00. 00000080:00000010:2.1:1775773076.713220:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb680. 00000080:00000010:0.1:1775773076.757382:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b7e0. 00010000:00000010:2.1:1775773076.788267:0:210358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00000080:00000010:3.1:1775773076.800838:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb680. 00010000:00000010:3.1:1775773076.800847:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3c00. 00010000:00000010:3.1:1775773076.800850:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00000080:00000010:2.1:1775773076.867352:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2ba0. 00000080:00000010:2.1:1775773076.876513:0:210361:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2040. 00010000:00000010:2.1:1775773076.913231:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3800. 00010000:00000010:2.1:1775773076.916597:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206187400. 00000080:00000010:2.1:1775773076.945480:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057ebc40. 00010000:00000010:2.1:1775773076.962912:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8400. 00000080:00000010:2.1:1775773076.980400:0:210362:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb960. 00000080:00000010:2.1:1775773076.993664:0:210363:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057ebb40. 00010000:00000010:2.1:1775773077.013578:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3f00. 00010000:00000010:2.1:1775773077.013586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8400. 00000080:00000010:2.1:1775773077.013591:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057ebb40. 00010000:00000010:2.1:1775773077.032227:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3000. 00010000:00000010:2.1:1775773077.032244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00000080:00000010:2.1:1775773077.032250:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb5c0. 00010000:00000010:2.1:1775773077.048180:0:210364:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3500. 00010000:00000010:2.1:1775773077.048445:0:210364:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:3.1:1775773077.072180:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcbd00. 00010000:00000010:3.1:1775773077.072208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89800. 00010000:00000010:3.1:1775773077.085362:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb700. 00000080:00000010:3.1:1775773077.105385:0:31:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62460. 00000080:00000010:3.1:1775773077.144303:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62660. 00010000:00000010:2.1:1775773077.170262:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3500. 00010000:00000010:2.1:1775773077.170318:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88200. 00000080:00000010:2.1:1775773077.183614:0:210366:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb8a0. 00010000:00000010:3.1:1775773077.222798:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89800. 00000080:00000010:3.1:1775773077.248177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62760. 00000080:00000010:3.1:1775773077.282306:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd628c0. 00010000:00000010:0.1:1775773077.307437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:3.1:1775773077.380185:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcbf00. 00010000:00000010:3.1:1775773077.380210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89800. 00010000:00000010:3.1:1775773077.397202:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb100. 00000080:00000010:0.1:1775773077.406185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd629c0. 00000080:00000010:0.1:1775773077.430924:0:2405:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd628c0. 00010000:00000010:3.1:1775773077.455244:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb100. 00010000:00000010:3.1:1775773077.455285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89a00. 00000080:00000010:3.1:1775773077.473372:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62f00. 00010000:00000010:2.1:1775773077.530613:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8400. 00000080:00000010:3.1:1775773077.604374:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62960. 00000080:00000010:2.1:1775773077.625092:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb0c0. 00010000:00000010:3.1:1775773077.661501:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb800. 00010000:00000010:3.1:1775773077.661514:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88600. 00000080:00000010:3.1:1775773077.661518:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62d00. 00010000:00000010:3.1:1775773077.667376:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb000. 00010000:00000010:3.1:1775773077.667385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88200. 00000080:00000010:3.1:1775773077.667388:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62e60. 00010000:00000010:2.1:1775773077.686479:0:210372:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3500. 00010000:00000010:2.1:1775773077.686494:0:210372:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8400. 00010000:00000010:2.1:1775773077.730177:0:210372:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3500. 00010000:00000010:2.1:1775773077.730198:0:210372:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8400. 00010000:00000010:2.1:1775773077.730202:0:210372:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3000. 00000080:00000010:2.1:1775773077.730205:0:210372:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057ebec0. 00000080:00000010:2.1:1775773077.739578:0:210373:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb0a0. 00010000:00000010:3.1:1775773077.775771:0:210374:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb000. 00010000:00000010:3.1:1775773077.775888:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9600. 00000080:00000010:3.1:1775773077.782336:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62040. 00010000:00000010:2.1:1775773077.809466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8400. 00000080:00000010:2.1:1775773077.815247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057ebe80. 00000080:00000010:2.1:1775773077.829286:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb0a0. 00010000:00000010:3.1:1775773077.865632:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88600. 00000080:00000010:2.1:1775773077.869286:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62220. 00010000:00000010:2.1:1775773077.869303:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0100. 00010000:00000010:2.1:1775773077.869309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb12e00. 00010000:00000010:3.1:1775773077.913400:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcbb00. 00010000:00000010:3.1:1775773077.913412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88600. 00010000:00000010:2.1:1775773077.932654:0:210376:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb900. 00000080:00000010:2.1:1775773077.950804:0:210377:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62e60. 00000080:00000010:2.1:1775773077.950832:0:210377:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62040. 00010000:00000010:3.1:1775773077.967510:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3500. 00010000:00000010:2.1:1775773077.977449:0:210378:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72200. 00000080:00000010:2.1:1775773077.993752:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb280. 00010000:00000010:2.1:1775773077.998804:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00000080:00000010:2.1:1775773078.016334:0:210379:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057ebdc0. 00000080:00000010:2.1:1775773078.029328:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057ebb20. 00010000:00000010:2.1:1775773078.049269:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3100. 00010000:00000010:2.1:1775773078.049282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:2.1:1775773078.060413:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202ae7f00. 00010000:00000010:2.1:1775773078.060423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206186e00. 00000080:00000010:2.1:1775773078.060428:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2800. 00010000:00000010:3.1:1775773078.080202:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcbb00. 00010000:00000010:3.1:1775773078.080213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73200. 00010000:00000010:2.1:1775773078.127510:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202ae7f00. 00010000:00000010:2.1:1775773078.127525:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206186e00. 00010000:00000010:2.1:1775773078.145289:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3100. 00000080:00000010:3.1:1775773078.152926:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057ebb20. 00000080:00000010:0.1:1775773078.160284:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2f00. 00010000:00000010:2.1:1775773078.185566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00000080:00000010:2.1:1775773078.200506:0:210382:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb280. 00010000:00000010:2.1:1775773078.234293:0:210382:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00000080:00000010:2.1:1775773078.249269:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb2e0. 00000080:00000010:2.1:1775773078.259492:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb280. 00010000:00000010:2.1:1775773078.277224:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202ae7c00. 00010000:00000010:2.1:1775773078.277239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206187e00. 00000080:00000010:2.1:1775773078.277244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2960. 00010000:00000010:2.1:1775773078.283550:0:210384:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3900. 00010000:00000010:2.1:1775773078.283559:0:210384:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335600. 00000080:00000010:2.1:1775773078.283562:0:210384:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb280. 00010000:00000010:2.1:1775773078.298298:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3500. 00010000:00000010:2.1:1775773078.298304:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335e00. 00010000:00000010:2.1:1775773078.308209:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3900. 00010000:00000010:2.1:1775773078.308220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335600. 00010000:00000010:3.1:1775773078.315428:0:210384:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3400. 00000080:00000010:3.1:1775773078.333360:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb0a0. 00000080:00000010:3.1:1775773078.333374:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb280. 00010000:00000010:3.1:1775773078.337317:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb900. 00010000:00000010:3.1:1775773078.337352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335c00. 00000080:00000010:3.1:1775773078.359199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62180. 00010000:00000010:2.1:1775773078.366412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335600. 00000080:00000010:2.1:1775773078.388469:0:210387:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb240. 00000080:00000010:2.1:1775773078.404339:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb2e0. 00010000:00000010:2.1:1775773078.421268:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3500. 00010000:00000010:2.1:1775773078.421275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335600. 00000080:00000010:2.1:1775773078.421279:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb240. 00010000:00000010:2.1:1775773078.463658:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3500. 00010000:00000010:2.1:1775773078.463671:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335600. 00010000:00000010:2.1:1775773078.471419:0:210388:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3c00. 00000080:00000010:2.1:1775773078.482655:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb2e0. 00000080:00000010:2.1:1775773078.482671:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb920. 00010000:00000010:2.1:1775773078.502417:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3c00. 00010000:00000010:3.1:1775773078.506483:0:210389:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e334c00. 00000080:00000010:3.1:1775773078.519423:0:210390:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd627a0. 00010000:00000010:3.1:1775773078.543434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72e00. 00000080:00000010:2.1:1775773078.551164:0:210390:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62ac0. 00000080:00000010:2.1:1775773078.564299:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd627a0. 00010000:00000010:2.1:1775773078.596478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb13200. 00000080:00000010:2.1:1775773078.613740:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b900. 00010000:00000010:2.1:1775773078.613779:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb800. 00010000:00000010:2.1:1775773078.613786:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89c00. 00010000:00000010:2.1:1775773078.671185:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3500. 00000080:00000010:3.1:1775773078.684180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62440. 00010000:00000010:2.1:1775773078.702184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb13600. 00000080:00000010:2.1:1775773078.712164:0:210393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb920. 00000080:00000010:2.1:1775773078.723393:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b320. 00010000:00000010:2.1:1775773078.755377:0:210395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e334200. 00000080:00000010:2.1:1775773078.771203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb2e0. 00010000:00000010:3.1:1775773078.796938:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72e00. 00000080:00000010:2.1:1775773078.812498:0:210396:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd629e0. 00000080:00000010:2.1:1775773078.812514:0:210396:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62fc0. 00010000:00000010:2.1:1775773078.843426:0:209610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72600. 00000080:00000010:2.1:1775773078.843439:0:209610:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb920. 00010000:00000010:2.1:1775773100.687181:0:211670:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb13600. 00010000:00000010:3.1:1775773100.785340:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0000. 00010000:00000010:3.1:1775773100.785347:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb12a00. 00000080:00000010:3.1:1775773100.794244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bd60. 00010000:00000010:0.1:1775773100.822283:0:211673:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb12e00. 00000080:00000010:0.1:1775773100.826705:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bb60. 00010000:00000010:0.1:1775773100.829311:0:211673:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb13400. 00000080:00000010:0.1:1775773100.829318:0:211673:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b2c0. 00010000:00000010:0.1:1775773100.843339:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb13400. 00010000:00000010:0.1:1775773100.847473:0:2405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb12e00. 00010000:00000010:2.1:1775773100.904408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335400. 00010000:00000010:2.1:1775773100.904420:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211939d00. 00010000:00000010:2.1:1775773100.904424:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335000. 00000080:00000010:2.1:1775773100.912204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505ee0. 00010000:00000010:2.1:1775773112.894213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211933c00. 00000080:00000010:2.1:1775773112.894228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505040. 00010000:00000010:3.1:1775773112.983261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201528000. 00010000:00000010:3.1:1775773112.996721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201528a00. 00000080:00000010:2.1:1775773113.000399:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62f60. 00010000:00000010:2.1:1775773113.000420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167fe00. 00010000:00000010:2.1:1775773113.066159:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211932c00. 00010000:00000010:3.1:1775773113.069329:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bed700. 00010000:00000010:3.1:1775773113.069341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211933c00. 00000080:00000010:3.1:1775773113.078354:0:211885:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505860. 00010000:00000010:3.1:1775773123.773174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb05800. 00000080:00000010:3.1:1775773123.773182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62c80. 00010000:00000010:0.1:1775773123.825622:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e70a000. 00010000:00000010:0.1:1775773123.834181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e70a600. 00000080:00000010:0.1:1775773123.834195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bc60. 00010000:00000010:0.1:1775773123.834204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233996200. 00010000:00000010:2.1:1775773123.890305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225937000. 00010000:00000010:2.1:1775773123.904208:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62120ca000. 00010000:00000010:2.1:1775773123.904221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb05800. 00000080:00000010:3.1:1775773123.909113:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62220. 00010000:00000010:0.1:1775773140.302180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62108b9200. 00000080:00000010:2.1:1775773140.302242:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233edb180. 00010000:00000010:3.1:1775773140.356914:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205728000. 00010000:00000010:3.1:1775773140.364353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205728600. 00000080:00000010:2.1:1775773140.369246:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62b20. 00010000:00000010:2.1:1775773140.369260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62108b9200. 00010000:00000010:2.1:1775773140.411187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dc09200. 00010000:00000010:2.1:1775773140.416181:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621193a600. 00010000:00000010:2.1:1775773140.416191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62108b9600. 00010000:00000010:2.1:1775773152.232171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623441a200. 00000080:00000010:2.1:1775773152.232179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93be0. 00010000:00000010:2.1:1775773152.270160:0:212513:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c706400. 00000080:00000010:2.1:1775773152.281359:0:212514:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4e20. 00010000:00000010:2.1:1775773152.281376:0:212514:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62066da200. 00010000:00000010:2.1:1775773152.309646:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623441b600. 00010000:00000010:2.1:1775773152.309654:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ccec900. 00010000:00000010:2.1:1775773152.309658:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623441ae00. 00000080:00000010:2.1:1775773152.326183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2f40. 00010000:00000010:2.1:1775773163.919272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c552600. 00000080:00000010:2.1:1775773163.923183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04580. 00010000:00000010:2.1:1775773163.968266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c552600. 00000080:00000010:2.1:1775773163.971315:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04c40. 00010000:00000010:2.1:1775773163.971329:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226477200. 00010000:00000010:3.1:1775773164.029580:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3e8c00. 00010000:00000010:3.1:1775773164.038242:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623386c400. 00010000:00000010:3.1:1775773164.038254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3e9600. 00000080:00000010:3.1:1775773164.045979:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b1b5b20. 00010000:00000010:3.1:1775773176.279406:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225804a00. 00000080:00000010:0.1:1775773176.279473:0:212931:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a80e0. 00010000:00000010:3.1:1775773176.356173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225804a00. 00000080:00000010:2.1:1775773176.375543:0:211424:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8780. 00010000:00000010:2.1:1775773176.375563:0:211424:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c934a00. 00010000:00000010:3.1:1775773176.475344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225804000. 00010000:00000010:3.1:1775773176.478302:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e122900. 00010000:00000010:3.1:1775773176.478318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225804a00. 00000080:00000010:2.1:1775773176.507195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8c00. 00010000:00000010:3.1:1775773188.579182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623491ce00. 00000080:00000010:3.1:1775773188.579201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233763e60. 00010000:00000010:2.1:1775773188.664217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623491d600. 00000080:00000010:2.1:1775773188.664241:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233763e80. 00010000:00000010:2.1:1775773188.664257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c605a00. 00010000:00000010:0.1:1775773188.719178:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265dde00. 00010000:00000010:0.1:1775773188.719186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c604200. 00000080:00000010:2.1:1775773188.738097:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbc40. 00010000:00000010:2.1:1775773202.623194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62082ce000. 00000080:00000010:2.1:1775773202.636178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15cc0. 00010000:00000010:3.1:1775773202.718213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca47000. 00000080:00000010:3.1:1775773202.723269:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a86e0. 00010000:00000010:3.1:1775773202.723291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202fc0a00. 00010000:00000010:2.1:1775773202.785324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335d9400. 00010000:00000010:2.1:1775773202.785339:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62262aba00. 00010000:00000010:2.1:1775773202.785343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262a8e00. 00000080:00000010:2.1:1775773202.809197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a157e0. 00010000:00000010:0.1:1775773216.274187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621836ae00. 00000080:00000010:0.1:1775773216.274193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15120. 00010000:00000010:0.1:1775773216.324164:0:213563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621836b600. 00000080:00000010:0.1:1775773216.330614:0:211424:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15c20. 00010000:00000010:0.1:1775773216.330631:0:211424:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6218368200. 00010000:00000010:2.1:1775773216.401182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd3b800. 00010000:00000010:3.1:1775773216.402271:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e3a00. 00010000:00000010:3.1:1775773216.402279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd3a600. 00000080:00000010:3.1:1775773216.412912:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f520. 00010000:00000010:2.1:1775773231.626359:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc4a000. 00000080:00000010:2.1:1775773231.631212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06c540. 00010000:00000010:2.1:1775773231.631228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b48c00. 00010000:00000010:0.1:1775773231.695188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b48c00. 00010000:00000010:0.1:1775773231.701077:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62346e6900. 00010000:00000010:0.1:1775773231.701092:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc4a800. 00000080:00000010:0.1:1775773231.718656:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06c500. 00010000:00000010:2.1:1775773243.135252:0:213986:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621839e800. 00010000:00000010:2.1:1775773243.135257:0:213986:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de6bf00. 00010000:00000010:2.1:1775773243.135260:0:213986:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621839f800. 00000080:00000010:2.1:1775773243.144412:0:213987:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8900. 00010000:00000010:3.1:1775773258.720182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8f7c00. 00000080:00000010:3.1:1775773258.720196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70720. 00010000:00000010:2.1:1775773258.770307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b5fc00. 00000080:00000010:2.1:1775773258.770341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380ac0. 00010000:00000010:2.1:1775773258.770350:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8f6e00. 00010000:00000010:3.1:1775773258.813511:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618fd00. 00010000:00000010:3.1:1775773258.813521:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b5fc00. 00010000:00000010:2.1:1775773258.814170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b5f400. 00000080:00000010:3.1:1775773258.826224:0:214198:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380c80. 00000800:00000200:3.1:1775773295.739301:0:2403:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 3, ksnr_connected 6 ksnr_max_conns 1 00000800:00000200:3.1:1775773295.739304:0:2403:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 2, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:3.1:1775773295.739307:0:2403:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 0 ksnr_max_conns 1 00000800:00000200:3.1:1775773385.019784:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773385.019992:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773385.020084:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773390.140101:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773390.140105:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775773390.140553:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773390.140556:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775773390.140887:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773390.140890:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775773400.379805:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773400.379810:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775773400.380191:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773400.380193:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775773400.380378:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773400.380380:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775773405.499755:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773405.499761:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775773405.500288:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773405.500291:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775773405.500612:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773405.500614:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775773410.619875:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773410.619882:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775773410.620416:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773410.620418:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775773410.620886:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773410.620889:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1775773431.099773:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773431.099779:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775773431.100200:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773431.100202:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775773431.100416:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773431.100418:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775773436.219590:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773436.219593:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775773436.219948:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773436.219950:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775773436.220240:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773436.220243:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775773441.339654:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773441.339657:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775773441.340000:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773441.340002:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775773441.340223:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773441.340224:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775773446.460276:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773446.460292:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775773446.460693:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773446.460695:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775773446.460912:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773446.460914:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775773466.940149:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773466.940153:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775773466.940773:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773466.940775:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775773466.941070:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773466.941072:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775773472.059810:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773472.059815:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775773472.060226:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773472.060229:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775773472.060499:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773472.060501:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775773477.179754:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773477.179758:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775773477.180333:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773477.180336:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775773477.180782:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773477.180785:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775773482.299874:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773482.299879:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775773482.300442:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773482.300445:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775773482.300674:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773482.300676:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1775773528.379796:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773528.379801:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775773528.380227:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773528.380229:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775773528.380438:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773528.380440:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000080:00000010:0.1:1775773680.069183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b160. 00000080:00000010:0.1:1775773680.074186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb2e0. 00000080:00000010:0.1:1775773680.074210:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7be00. 00000080:00000010:0.1:1775773680.074226:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb7c0. 00000080:00000010:0.1:1775773680.074308:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb0c0. 00000080:00000010:0.1:1775773680.074400:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62600. 00000080:00000010:0.1:1775773680.074481:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b180. 00000080:00000010:0.1:1775773680.074563:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057ebf60. 00000080:00000010:0.1:1775773680.074628:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb020. 00000080:00000010:0.1:1775773680.078166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057ebac0. 00000080:00000010:0.1:1775773680.078194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb380. 00000080:00000010:0.1:1775773680.078198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd621a0. 00010000:00000010:0.1:1775773680.078217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335200. 00010000:00000010:0.1:1775773680.078221:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211939200. 00010000:00000010:0.1:1775773680.078225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb12000. 00010000:00000010:0.1:1775773680.078228:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb800. 00010000:00000010:0.1:1775773680.078231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e334200. 00010000:00000010:0.1:1775773680.078233:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211939e00. 00010000:00000010:0.1:1775773680.078236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb13c00. 00010000:00000010:0.1:1775773680.078238:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0200. 00010000:00000010:0.1:1775773680.078240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb13000. 00010000:00000010:0.1:1775773680.078244:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3c00. 00010000:00000010:0.1:1775773680.078246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e334600. 00010000:00000010:0.1:1775773680.078249:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb900. 00010000:00000010:0.1:1775773680.078251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73000. 00010000:00000010:0.1:1775773680.078254:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3400. 00010000:00000010:0.1:1775773680.078256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335600. 00010000:00000010:0.1:1775773680.078258:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3900. 00010000:00000010:0.1:1775773680.078261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335e00. 00010000:00000010:0.1:1775773680.078263:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcbb00. 00010000:00000010:0.1:1775773680.078264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73200. 00010000:00000010:0.1:1775773680.078266:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3100. 00010000:00000010:0.1:1775773680.078268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e334400. 00010000:00000010:0.1:1775773680.078270:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3600. 00010000:00000010:0.1:1775773680.078273:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb12400. 00010000:00000010:0.1:1775773680.078275:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3b00. 00010000:00000010:0.1:1775773680.078278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9400. 00010000:00000010:0.1:1775773680.078281:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3000. 00010000:00000010:0.1:1775773680.078283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8400. 00010000:00000010:0.1:1775773680.078285:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb700. 00010000:00000010:0.1:1775773680.078288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88e00. 00010000:00000010:0.1:1775773680.078290:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcbd00. 00010000:00000010:0.1:1775773680.078292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9800. 00010000:00000010:0.1:1775773680.078294:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3800. 00010000:00000010:0.1:1775773680.078295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8600. 00010000:00000010:0.1:1775773680.078297:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202ae7600. 00010000:00000010:0.1:1775773680.078300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9000. 00010000:00000010:0.1:1775773680.078302:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcba00. 00010000:00000010:0.1:1775773680.078303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89000. 00010000:00000010:0.1:1775773680.078305:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202ae7b00. 00010000:00000010:0.1:1775773680.078307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8000. 00010000:00000010:0.1:1775773680.078309:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3e00. 00010000:00000010:0.1:1775773680.078311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8200. 00010000:00000010:0.1:1775773680.078313:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3700. 00010000:00000010:0.1:1775773680.078315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f9a00. 00010000:00000010:0.1:1775773680.078317:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bcb400. 00010000:00000010:0.1:1775773680.078318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88a00. 00000800:00000200:3.1:1775773702.460009:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773702.460015:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775773702.460401:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773702.460403:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775773702.460593:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773702.460596:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775773707.580422:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773707.580427:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775773707.580971:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773707.580974:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775773707.581406:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773707.581409:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000080:00000010:0.1:1775773710.834163:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2100. 00000080:00000010:0.1:1775773710.834175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a26e0. 00000080:00000010:0.1:1775773710.834178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2180. 00000080:00000010:0.1:1775773710.834183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b840. 00000080:00000010:0.1:1775773710.838177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5051e0. 00000080:00000010:0.1:1775773710.838190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505260. 00000080:00000010:0.1:1775773710.838196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bfe0. 00000080:00000010:0.1:1775773710.838202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2d20. 00000080:00000010:0.1:1775773710.838208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5054e0. 00000080:00000010:0.1:1775773710.838214:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505660. 00000080:00000010:0.1:1775773710.838220:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505ca0. 00000080:00000010:0.1:1775773710.838225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505800. 00000080:00000010:0.1:1775773710.838235:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b520. 00000080:00000010:0.1:1775773710.838241:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505400. 00000080:00000010:0.1:1775773710.838246:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505120. 00000080:00000010:0.1:1775773710.838251:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505e00. 00000080:00000010:0.1:1775773710.838256:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bd40. 00000080:00000010:0.1:1775773710.838261:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505980. 00000080:00000010:0.1:1775773710.838267:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5059e0. 00000080:00000010:0.1:1775773710.838275:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505a40. 00000080:00000010:0.1:1775773710.838280:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505ba0. 00000080:00000010:0.1:1775773710.838285:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5053a0. 00000080:00000010:0.1:1775773710.838290:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5053c0. 00000080:00000010:0.1:1775773710.838294:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5056e0. 00000080:00000010:0.1:1775773710.838301:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505540. 00000080:00000010:0.1:1775773710.838308:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5056a0. 00000080:00000010:0.1:1775773710.838311:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505a60. 00000080:00000010:0.1:1775773710.838315:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505020. 00000080:00000010:0.1:1775773710.838318:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505ae0. 00000080:00000010:0.1:1775773710.838321:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505f20. 00000080:00000010:0.1:1775773710.838325:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bd20. 00000080:00000010:0.1:1775773710.838328:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505280. 00000080:00000010:0.1:1775773710.838331:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505900. 00000080:00000010:0.1:1775773710.838335:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5058a0. 00000080:00000010:0.1:1775773710.838338:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505d60. 00000080:00000010:0.1:1775773710.838341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a23e0. 00000080:00000010:0.1:1775773710.838345:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505f60. 00000080:00000010:0.1:1775773710.838348:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b5e0. 00000080:00000010:0.1:1775773710.838351:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7baa0. 00000080:00000010:0.1:1775773710.838355:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd626a0. 00000080:00000010:0.1:1775773710.838359:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2460. 00000080:00000010:0.1:1775773710.838361:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2c40. 00000080:00000010:0.1:1775773710.838364:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62520. 00000080:00000010:0.1:1775773710.838367:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505600. 00000080:00000010:0.1:1775773710.838370:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505be0. 00000080:00000010:0.1:1775773710.838373:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5057a0. 00000080:00000010:0.1:1775773710.838377:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505da0. 00000080:00000010:0.1:1775773710.838380:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b660. 00000080:00000010:0.1:1775773710.838383:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5059a0. 00000080:00000010:0.1:1775773710.838389:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505760. 00000080:00000010:0.1:1775773710.838392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5051a0. 00000080:00000010:0.1:1775773710.838395:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505e60. 00000080:00000010:0.1:1775773710.838398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd627c0. 00000080:00000010:0.1:1775773710.838401:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5058c0. 00000080:00000010:0.1:1775773710.838403:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505cc0. 00000080:00000010:0.1:1775773710.838406:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bac0. 00000080:00000010:0.1:1775773710.838409:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505f80. 00000080:00000010:0.1:1775773710.838412:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b540. 00000080:00000010:0.1:1775773710.838414:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5057e0. 00000080:00000010:0.1:1775773710.838417:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505620. 00000080:00000010:0.1:1775773710.838419:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505a00. 00000080:00000010:0.1:1775773710.838422:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505000. 00000080:00000010:0.1:1775773710.838424:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a29c0. 00000080:00000010:0.1:1775773710.838428:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b100. 00000080:00000010:0.1:1775773710.838431:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2080. 00000080:00000010:0.1:1775773710.838434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505d40. 00000080:00000010:0.1:1775773710.838439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a22c0. 00000080:00000010:0.1:1775773710.838442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2720. 00000080:00000010:0.1:1775773710.838444:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2520. 00000080:00000010:0.1:1775773710.838447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bdc0. 00000080:00000010:0.1:1775773710.838449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bb40. 00000080:00000010:0.1:1775773710.838452:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7ba00. 00000080:00000010:0.1:1775773710.838455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5056c0. 00000080:00000010:0.1:1775773710.838458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505ce0. 00000080:00000010:0.1:1775773710.838461:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505460. 00000080:00000010:0.1:1775773710.838464:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505220. 00000080:00000010:0.1:1775773710.838467:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505880. 00000080:00000010:0.1:1775773710.838470:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505240. 00000080:00000010:0.1:1775773710.838472:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505d20. 00000080:00000010:0.1:1775773710.838475:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505a20. 00000080:00000010:0.1:1775773710.838477:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bf80. 00000080:00000010:0.1:1775773710.838480:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505ea0. 00000080:00000010:0.1:1775773710.838483:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505140. 00000080:00000010:0.1:1775773710.838486:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505580. 00000080:00000010:0.1:1775773710.838488:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505380. 00000080:00000010:0.1:1775773710.838491:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505fe0. 00000080:00000010:0.1:1775773710.838494:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505b80. 00000080:00000010:0.1:1775773710.838496:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505300. 00000080:00000010:0.1:1775773710.838499:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505c80. 00000080:00000010:0.1:1775773710.838503:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd621c0. 00000080:00000010:0.1:1775773710.838506:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505320. 00000080:00000010:0.1:1775773710.838508:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505060. 00000080:00000010:0.1:1775773710.838511:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b440. 00000080:00000010:0.1:1775773710.838513:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b620. 00000080:00000010:0.1:1775773710.838515:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b640. 00000080:00000010:0.1:1775773710.838518:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5057c0. 00000080:00000010:0.1:1775773710.838521:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505dc0. 00000080:00000010:0.1:1775773710.838524:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7be60. 00000080:00000010:0.1:1775773710.838526:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a28a0. 00000080:00000010:0.1:1775773710.838529:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd629c0. 00000080:00000010:0.1:1775773710.838531:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505f00. 00000080:00000010:0.1:1775773710.838534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505e40. 00000080:00000010:0.1:1775773710.838536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505500. 00000080:00000010:0.1:1775773710.838539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2300. 00000080:00000010:0.1:1775773710.838541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505440. 00000080:00000010:0.1:1775773710.838544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5055c0. 00000080:00000010:0.1:1775773710.838546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bba0. 00000080:00000010:0.1:1775773710.838549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bb60. 00000080:00000010:0.1:1775773710.838552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505c00. 00000080:00000010:0.1:1775773710.838555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505940. 00000080:00000010:0.1:1775773710.838558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505aa0. 00000080:00000010:0.1:1775773710.838562:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2660. 00000080:00000010:0.1:1775773710.838565:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505200. 00000080:00000010:0.1:1775773710.838568:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505b40. 00000080:00000010:0.1:1775773710.838570:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62de0. 00000080:00000010:0.1:1775773710.838573:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5054a0. 00000080:00000010:0.1:1775773710.838576:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd622e0. 00000080:00000010:0.1:1775773710.838578:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5052c0. 00000080:00000010:0.1:1775773710.838581:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b120. 00000080:00000010:0.1:1775773710.838584:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505de0. 00000080:00000010:0.1:1775773710.838587:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62cc0. 00000080:00000010:0.1:1775773710.838589:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505b20. 00000080:00000010:0.1:1775773710.838592:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2c60. 00000080:00000010:0.1:1775773710.838595:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2a00. 00000080:00000010:0.1:1775773710.838597:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2c80. 00000080:00000010:0.1:1775773710.838599:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62080. 00000080:00000010:0.1:1775773710.838602:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505d00. 00000080:00000010:0.1:1775773710.838604:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bd80. 00000080:00000010:0.1:1775773710.838606:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505fc0. 00000080:00000010:0.1:1775773710.838608:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b4a0. 00000080:00000010:0.1:1775773710.838611:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505a80. 00000080:00000010:0.1:1775773710.838613:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62920. 00000080:00000010:0.1:1775773710.838615:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62ea0. 00000080:00000010:0.1:1775773710.838618:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505420. 00000080:00000010:0.1:1775773710.838621:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5055e0. 00000080:00000010:0.1:1775773710.838623:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505780. 00000080:00000010:0.1:1775773710.838625:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505e80. 00000080:00000010:0.1:1775773710.838627:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62440. 00000080:00000010:0.1:1775773710.838630:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62360. 00000080:00000010:0.1:1775773710.838633:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505ee0. 00000080:00000010:0.1:1775773710.838636:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505b60. 00000080:00000010:0.1:1775773710.838638:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505720. 00000080:00000010:0.1:1775773710.838641:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505480. 00000080:00000010:0.1:1775773710.838643:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b8c0. 00000080:00000010:0.1:1775773710.838645:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62860. 00000080:00000010:0.1:1775773710.838647:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505820. 00000080:00000010:0.1:1775773710.838650:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5051c0. 00000080:00000010:0.1:1775773710.838652:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62300. 00000080:00000010:0.1:1775773710.838654:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505080. 00000080:00000010:0.1:1775773710.838656:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5050e0. 00000080:00000010:0.1:1775773710.838658:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505d80. 00000080:00000010:0.1:1775773710.838660:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505360. 00000080:00000010:0.1:1775773710.838663:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62960. 00000080:00000010:0.1:1775773710.838665:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505160. 00000080:00000010:0.1:1775773710.838669:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62940. 00000080:00000010:0.1:1775773710.838671:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62dc0. 00000080:00000010:0.1:1775773710.838673:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2aa0. 00000080:00000010:0.1:1775773710.838676:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2d60. 00000080:00000010:0.1:1775773710.838678:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b2c0. 00000080:00000010:0.1:1775773710.838681:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505ec0. 00000080:00000010:0.1:1775773710.838683:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5059c0. 00000080:00000010:0.1:1775773710.838685:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bd00. 00000080:00000010:0.1:1775773710.838687:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62e00. 00000080:00000010:0.1:1775773710.838689:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505700. 00000080:00000010:0.1:1775773710.838692:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5052e0. 00000080:00000010:0.1:1775773710.838694:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5053e0. 00000080:00000010:0.1:1775773710.838696:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b280. 00000080:00000010:0.1:1775773710.838698:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505c20. 00000080:00000010:0.1:1775773710.838700:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b6c0. 00000080:00000010:0.1:1775773710.838702:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505680. 00000080:00000010:0.1:1775773710.838704:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b820. 00000080:00000010:0.1:1775773710.838706:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505fa0. 00000080:00000010:0.1:1775773710.838708:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505180. 00000080:00000010:0.1:1775773710.838710:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c5050a0. 00000080:00000010:0.1:1775773710.838712:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505b00. 00000080:00000010:0.1:1775773710.838714:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62fa0. 00000080:00000010:0.1:1775773710.838716:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505740. 00000080:00000010:0.1:1775773710.838718:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505840. 00000080:00000010:0.1:1775773710.838720:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2620. 00000080:00000010:0.1:1775773710.838722:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505920. 00000080:00000010:0.1:1775773710.838724:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505640. 00000080:00000010:0.1:1775773710.838726:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a27e0. 00000080:00000010:0.1:1775773710.838728:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2cc0. 00000080:00000010:0.1:1775773710.838730:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505bc0. 00010000:00000010:0.1:1775773710.838733:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62347ffb00. 00010000:00000010:0.1:1775773710.838736:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200030800. 00010000:00000010:0.1:1775773710.838738:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206551200. 00010000:00000010:0.1:1775773710.838740:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200031800. 00010000:00000010:0.1:1775773710.838742:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206551300. 00010000:00000010:0.1:1775773710.838745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200031000. 00010000:00000010:0.1:1775773710.838747:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c931a00. 00010000:00000010:0.1:1775773710.838749:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167f000. 00010000:00000010:0.1:1775773710.838752:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205e00. 00010000:00000010:0.1:1775773710.838754:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211933a00. 00010000:00000010:0.1:1775773710.838756:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6212fbb500. 00010000:00000010:0.1:1775773710.838759:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211932e00. 00010000:00000010:0.1:1775773710.838761:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205300. 00010000:00000010:0.1:1775773710.838763:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167ea00. 00010000:00000010:0.1:1775773710.838764:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205200. 00010000:00000010:0.1:1775773710.838766:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200030a00. 00010000:00000010:0.1:1775773710.838768:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205d00. 00010000:00000010:0.1:1775773710.838770:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211933600. 00010000:00000010:0.1:1775773710.838771:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6212fbb300. 00010000:00000010:0.1:1775773710.838773:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211932000. 00010000:00000010:0.1:1775773710.838775:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205900. 00010000:00000010:0.1:1775773710.838776:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211932a00. 00010000:00000010:0.1:1775773710.838778:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205700. 00010000:00000010:0.1:1775773710.838779:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211933400. 00010000:00000010:0.1:1775773710.838781:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6212fbb800. 00010000:00000010:0.1:1775773710.838783:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167e200. 00010000:00000010:0.1:1775773710.838784:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205800. 00010000:00000010:0.1:1775773710.838786:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211932200. 00010000:00000010:0.1:1775773710.838787:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205a00. 00010000:00000010:0.1:1775773710.838789:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226005c00. 00010000:00000010:0.1:1775773710.838791:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203205c00. 00010000:00000010:0.1:1775773710.838793:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226004e00. 00010000:00000010:0.1:1775773710.838795:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9a00. 00010000:00000010:0.1:1775773710.838797:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167fc00. 00010000:00000010:0.1:1775773710.838799:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c931100. 00010000:00000010:0.1:1775773710.838800:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226004200. 00010000:00000010:0.1:1775773710.838802:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9700. 00010000:00000010:0.1:1775773710.838805:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226005a00. 00010000:00000010:0.1:1775773710.838806:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206551b00. 00010000:00000010:0.1:1775773710.838808:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226005400. 00010000:00000010:0.1:1775773710.838810:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9900. 00010000:00000010:0.1:1775773710.838812:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226005600. 00010000:00000010:0.1:1775773710.838814:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9800. 00010000:00000010:0.1:1775773710.838816:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226004a00. 00010000:00000010:0.1:1775773710.838817:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9200. 00010000:00000010:0.1:1775773710.838819:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226004c00. 00010000:00000010:0.1:1775773710.838820:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9f00. 00010000:00000010:0.1:1775773710.838822:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226005200. 00010000:00000010:0.1:1775773710.838823:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206551f00. 00010000:00000010:0.1:1775773710.838825:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226005000. 00010000:00000010:0.1:1775773710.838827:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9300. 00010000:00000010:0.1:1775773710.838828:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226004400. 00010000:00000010:0.1:1775773710.838830:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9100. 00010000:00000010:0.1:1775773710.838831:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226004600. 00010000:00000010:0.1:1775773710.838833:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9500. 00010000:00000010:0.1:1775773710.838834:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226004000. 00010000:00000010:0.1:1775773710.838836:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9d00. 00010000:00000010:0.1:1775773710.838838:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226005e00. 00010000:00000010:0.1:1775773710.838839:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9c00. 00010000:00000010:0.1:1775773710.838841:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226005800. 00010000:00000010:0.1:1775773710.838843:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206551d00. 00010000:00000010:0.1:1775773710.838844:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167ee00. 00010000:00000010:0.1:1775773710.838846:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9e00. 00010000:00000010:0.1:1775773710.838847:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226004800. 00010000:00000010:0.1:1775773710.838849:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9000. 00010000:00000010:0.1:1775773710.838851:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46da00. 00010000:00000010:0.1:1775773710.838853:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9600. 00010000:00000010:0.1:1775773710.838855:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46c200. 00010000:00000010:0.1:1775773710.838856:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9b00. 00010000:00000010:0.1:1775773710.838858:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46ce00. 00010000:00000010:0.1:1775773710.838860:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6212fbbc00. 00010000:00000010:0.1:1775773710.838861:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200031200. 00010000:00000010:0.1:1775773710.838863:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349c9400. 00010000:00000010:0.1:1775773710.838865:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46d600. 00010000:00000010:0.1:1775773710.838867:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6212fbb100. 00010000:00000010:0.1:1775773710.838869:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167e400. 00010000:00000010:0.1:1775773710.838870:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6212fbb600. 00010000:00000010:0.1:1775773710.838872:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620167f800. 00010000:00000010:0.1:1775773710.838874:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c931700. 00010000:00000010:0.1:1775773710.838877:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6201529600. 00010000:00000010:0.1:1775773710.838880:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6212fbb200. 00010000:00000010:0.1:1775773710.838883:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200031e00. 00010000:00000010:0.1:1775773710.838886:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b89200. 00010000:00000010:0.1:1775773710.838889:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200030400. 00010000:00000010:0.1:1775773710.838892:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c931c00. 00010000:00000010:0.1:1775773710.838894:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dc400. 00010000:00000010:0.1:1775773710.838897:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b89e00. 00010000:00000010:0.1:1775773710.838900:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46cc00. 00010000:00000010:0.1:1775773710.838903:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202ae7f00. 00010000:00000010:0.1:1775773710.838906:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46dc00. 00010000:00000010:0.1:1775773710.838909:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c931300. 00010000:00000010:0.1:1775773710.838912:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46d400. 00010000:00000010:0.1:1775773710.838915:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b89700. 00010000:00000010:0.1:1775773710.838918:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46ca00. 00010000:00000010:0.1:1775773710.838920:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6212fbb700. 00010000:00000010:0.1:1775773710.838924:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101126e00. 00010000:00000010:0.1:1775773710.838927:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b89f00. 00010000:00000010:0.1:1775773710.838930:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46c000. 00010000:00000010:0.1:1775773710.838933:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c931500. 00010000:00000010:0.1:1775773710.838935:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46c600. 00010000:00000010:0.1:1775773710.838938:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b89c00. 00010000:00000010:0.1:1775773710.838940:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46d800. 00010000:00000010:0.1:1775773710.838942:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b89400. 00010000:00000010:0.1:1775773710.838944:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46c800. 00010000:00000010:0.1:1775773710.838945:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c931400. 00010000:00000010:0.1:1775773710.838947:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dc800. 00010000:00000010:0.1:1775773710.838949:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b89100. 00010000:00000010:0.1:1775773710.838950:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46c400. 00010000:00000010:0.1:1775773710.838952:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b89500. 00010000:00000010:0.1:1775773710.838953:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46d200. 00010000:00000010:0.1:1775773710.838956:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203a46300. 00010000:00000010:0.1:1775773710.838958:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101127800. 00010000:00000010:0.1:1775773710.838960:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203a46600. 00010000:00000010:0.1:1775773710.838962:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46de00. 00010000:00000010:0.1:1775773710.838964:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203a46000. 00010000:00000010:0.1:1775773710.838966:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101126200. 00010000:00000010:0.1:1775773710.838969:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b89800. 00010000:00000010:0.1:1775773710.838971:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c46d000. 00010000:00000010:0.1:1775773710.838974:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b89d00. 00010000:00000010:0.1:1775773710.838976:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206765a00. 00010000:00000010:0.1:1775773710.838979:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b89a00. 00010000:00000010:0.1:1775773710.838982:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206764e00. 00010000:00000010:0.1:1775773710.838985:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b89000. 00010000:00000010:0.1:1775773710.838988:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206765600. 00010000:00000010:0.1:1775773710.838991:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b89300. 00010000:00000010:0.1:1775773710.838994:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6200030000. 00010000:00000010:0.1:1775773710.838997:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b89900. 00010000:00000010:0.1:1775773710.839000:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101126000. 00010000:00000010:0.1:1775773710.839003:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b89600. 00010000:00000010:0.1:1775773710.839006:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206186200. 00010000:00000010:0.1:1775773710.839009:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b89b00. 00010000:00000010:0.1:1775773710.839012:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206764c00. 00010000:00000010:0.1:1775773710.839015:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e40b400. 00010000:00000010:0.1:1775773710.839019:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206186c00. 00010000:00000010:0.1:1775773710.839021:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6f400. 00010000:00000010:0.1:1775773710.839023:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206187200. 00010000:00000010:0.1:1775773710.839025:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203a46900. 00010000:00000010:0.1:1775773710.839027:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206186e00. 00010000:00000010:0.1:1775773710.839029:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203a46400. 00010000:00000010:0.1:1775773710.839031:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101126400. 00010000:00000010:0.1:1775773710.839033:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203a46800. 00010000:00000010:0.1:1775773710.839035:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101127400. 00010000:00000010:0.1:1775773710.839037:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203a46500. 00010000:00000010:0.1:1775773710.839038:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101126800. 00010000:00000010:0.1:1775773710.839040:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e40b500. 00010000:00000010:0.1:1775773710.839041:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206765c00. 00010000:00000010:0.1:1775773710.839043:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e40b200. 00010000:00000010:0.1:1775773710.839045:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206765400. 00010000:00000010:0.1:1775773710.839046:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e40bf00. 00010000:00000010:0.1:1775773710.839048:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206764a00. 00010000:00000010:0.1:1775773710.839050:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e40be00. 00010000:00000010:0.1:1775773710.839051:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206764000. 00010000:00000010:0.1:1775773710.839053:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e40b100. 00010000:00000010:0.1:1775773710.839055:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206764600. 00010000:00000010:0.1:1775773710.839056:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e40bc00. 00010000:00000010:0.1:1775773710.839058:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206765800. 00010000:00000010:0.1:1775773710.839060:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e40b700. 00010000:00000010:0.1:1775773710.839061:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206764800. 00010000:00000010:0.1:1775773710.839063:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e40b000. 00010000:00000010:0.1:1775773710.839064:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206764400. 00010000:00000010:0.1:1775773710.839066:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202ae7c00. 00010000:00000010:0.1:1775773710.839067:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101127000. 00010000:00000010:0.1:1775773710.839069:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e40ba00. 00010000:00000010:0.1:1775773710.839071:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206765200. 00010000:00000010:0.1:1775773710.839072:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e40b900. 00010000:00000010:0.1:1775773710.839074:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206765e00. 00010000:00000010:0.1:1775773710.839076:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e40b800. 00010000:00000010:0.1:1775773710.839078:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206765000. 00010000:00000010:0.1:1775773710.839080:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e40bb00. 00010000:00000010:0.1:1775773710.839081:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206764200. 00010000:00000010:0.1:1775773710.839084:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e800. 00010000:00000010:0.1:1775773710.839086:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348be000. 00010000:00000010:0.1:1775773710.839087:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e40bd00. 00010000:00000010:0.1:1775773710.839089:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bea00. 00010000:00000010:0.1:1775773710.839091:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e40b600. 00010000:00000010:0.1:1775773710.839092:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348be800. 00010000:00000010:0.1:1775773710.839094:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e500. 00010000:00000010:0.1:1775773710.839095:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348be400. 00010000:00000010:0.1:1775773710.839097:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6f000. 00010000:00000010:0.1:1775773710.839098:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dd200. 00010000:00000010:0.1:1775773710.839100:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e40b300. 00010000:00000010:0.1:1775773710.839102:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348be600. 00010000:00000010:0.1:1775773710.839116:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fb000. 00010000:00000010:0.1:1775773710.839119:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bf200. 00010000:00000010:0.1:1775773710.839121:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e200. 00010000:00000010:0.1:1775773710.839122:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101127200. 00010000:00000010:0.1:1775773710.839124:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fb800. 00010000:00000010:0.1:1775773710.839126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101126600. 00010000:00000010:0.1:1775773710.839127:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388eb00. 00010000:00000010:0.1:1775773710.839129:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101127c00. 00010000:00000010:0.1:1775773710.839130:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388ee00. 00010000:00000010:0.1:1775773710.839132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bfe00. 00010000:00000010:0.1:1775773710.839147:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fba00. 00010000:00000010:0.1:1775773710.839149:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bfa00. 00010000:00000010:0.1:1775773710.839151:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203a46f00. 00010000:00000010:0.1:1775773710.839153:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101127a00. 00010000:00000010:0.1:1775773710.839155:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388ed00. 00010000:00000010:0.1:1775773710.839157:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206187e00. 00010000:00000010:0.1:1775773710.839159:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6f500. 00010000:00000010:0.1:1775773710.839161:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dc000. 00010000:00000010:0.1:1775773710.839163:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fbe00. 00010000:00000010:0.1:1775773710.839165:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bf000. 00010000:00000010:0.1:1775773710.839167:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fb600. 00010000:00000010:0.1:1775773710.839168:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348be200. 00010000:00000010:0.1:1775773710.839170:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fb300. 00010000:00000010:0.1:1775773710.839171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bfc00. 00010000:00000010:0.1:1775773710.839173:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fb500. 00010000:00000010:0.1:1775773710.839175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206186800. 00010000:00000010:0.1:1775773710.839176:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fb200. 00010000:00000010:0.1:1775773710.839178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bec00. 00010000:00000010:0.1:1775773710.839179:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fb700. 00010000:00000010:0.1:1775773710.839181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bf400. 00010000:00000010:0.1:1775773710.839183:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203a46e00. 00010000:00000010:0.1:1775773710.839184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101127e00. 00010000:00000010:0.1:1775773710.839186:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0000. 00010000:00000010:0.1:1775773710.839188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb12a00. 00010000:00000010:0.1:1775773710.839190:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fbf00. 00010000:00000010:0.1:1775773710.839192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bf600. 00010000:00000010:0.1:1775773710.839193:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fb400. 00010000:00000010:0.1:1775773710.839195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bee00. 00010000:00000010:0.1:1775773710.839196:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fbb00. 00010000:00000010:0.1:1775773710.839198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62348bf800. 00010000:00000010:0.1:1775773710.839199:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6fe00. 00010000:00000010:0.1:1775773710.839201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206186000. 00010000:00000010:0.1:1775773710.839203:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fb100. 00010000:00000010:0.1:1775773710.839205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4ac00. 00010000:00000010:0.1:1775773710.839207:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6fd00. 00010000:00000010:0.1:1775773710.839208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4a200. 00010000:00000010:0.1:1775773710.839210:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6fa00. 00010000:00000010:0.1:1775773710.839212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dc600. 00010000:00000010:0.1:1775773710.839214:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fbc00. 00010000:00000010:0.1:1775773710.839216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4b600. 00010000:00000010:0.1:1775773710.839218:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6f700. 00010000:00000010:0.1:1775773710.839219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dd000. 00010000:00000010:0.1:1775773710.839221:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6f300. 00010000:00000010:0.1:1775773710.839223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62421dde00. 00010000:00000010:0.1:1775773710.839224:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0b00. 00010000:00000010:0.1:1775773710.839226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb12800. 00010000:00000010:0.1:1775773710.839227:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fbd00. 00010000:00000010:0.1:1775773710.839229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4aa00. 00010000:00000010:0.1:1775773710.839231:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0300. 00010000:00000010:0.1:1775773710.839232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb12600. 00010000:00000010:0.1:1775773710.839234:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107fb900. 00010000:00000010:0.1:1775773710.839235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4be00. 00010000:00000010:0.1:1775773710.839237:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388ef00. 00010000:00000010:0.1:1775773710.839238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206187a00. 00010000:00000010:0.1:1775773710.839240:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e600. 00010000:00000010:0.1:1775773710.839242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206186600. 00010000:00000010:0.1:1775773710.839243:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e100. 00010000:00000010:0.1:1775773710.839245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206187000. 00010000:00000010:0.1:1775773710.839246:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6f900. 00010000:00000010:0.1:1775773710.839248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce88600. 00010000:00000010:0.1:1775773710.839250:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211364c00. 00010000:00000010:0.1:1775773710.839252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4a000. 00010000:00000010:0.1:1775773710.839253:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0700. 00010000:00000010:0.1:1775773710.839255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb13200. 00010000:00000010:0.1:1775773710.839257:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0c00. 00010000:00000010:0.1:1775773710.839259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4ae00. 00010000:00000010:0.1:1775773710.839260:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0500. 00010000:00000010:0.1:1775773710.839262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb13e00. 00010000:00000010:0.1:1775773710.839264:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211364000. 00010000:00000010:0.1:1775773710.839266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4ba00. 00010000:00000010:0.1:1775773710.839267:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211a6f800. 00010000:00000010:0.1:1775773710.839269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce89c00. 00010000:00000010:0.1:1775773710.839272:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990300. 00010000:00000010:0.1:1775773710.839274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73400. 00010000:00000010:0.1:1775773710.839276:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211364200. 00010000:00000010:0.1:1775773710.839278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4b400. 00010000:00000010:0.1:1775773710.839279:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211364e00. 00010000:00000010:0.1:1775773710.839281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4a600. 00010000:00000010:0.1:1775773710.839282:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211364b00. 00010000:00000010:0.1:1775773710.839284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4a800. 00010000:00000010:0.1:1775773710.839286:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211364f00. 00010000:00000010:0.1:1775773710.839288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4b200. 00010000:00000010:0.1:1775773710.839289:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990b00. 00010000:00000010:0.1:1775773710.839291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72a00. 00010000:00000010:0.1:1775773710.839293:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990200. 00010000:00000010:0.1:1775773710.839294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73a00. 00010000:00000010:0.1:1775773710.839296:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211364600. 00010000:00000010:0.1:1775773710.839297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4b800. 00010000:00000010:0.1:1775773710.839299:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211364d00. 00010000:00000010:0.1:1775773710.839301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4a400. 00010000:00000010:0.1:1775773710.839302:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211364900. 00010000:00000010:0.1:1775773710.839304:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4b000. 00010000:00000010:0.1:1775773710.839305:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211364a00. 00010000:00000010:0.1:1775773710.839307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f610fc4bc00. 00010000:00000010:0.1:1775773710.839308:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211364100. 00010000:00000010:0.1:1775773710.839310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb13400. 00010000:00000010:0.1:1775773710.839312:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990d00. 00010000:00000010:0.1:1775773710.839313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73800. 00010000:00000010:0.1:1775773710.839315:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211364400. 00010000:00000010:0.1:1775773710.839317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101124c00. 00010000:00000010:0.1:1775773710.839319:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211364700. 00010000:00000010:0.1:1775773710.839322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101125800. 00010000:00000010:0.1:1775773710.839323:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211364300. 00010000:00000010:0.1:1775773710.839325:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73c00. 00010000:00000010:0.1:1775773710.839327:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0800. 00010000:00000010:0.1:1775773710.839329:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101124e00. 00010000:00000010:0.1:1775773710.839331:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990f00. 00010000:00000010:0.1:1775773710.839333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101125600. 00010000:00000010:0.1:1775773710.839334:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388ea00. 00010000:00000010:0.1:1775773710.839336:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101125a00. 00010000:00000010:0.1:1775773710.839337:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211364500. 00010000:00000010:0.1:1775773710.839339:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101124200. 00010000:00000010:0.1:1775773710.839340:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990000. 00010000:00000010:0.1:1775773710.839342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72400. 00010000:00000010:0.1:1775773710.839343:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990900. 00010000:00000010:0.1:1775773710.839345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101125000. 00010000:00000010:0.1:1775773710.839346:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990100. 00010000:00000010:0.1:1775773710.839348:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72800. 00010000:00000010:0.1:1775773710.839350:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990500. 00010000:00000010:0.1:1775773710.839351:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da73600. 00010000:00000010:0.1:1775773710.839353:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e400. 00010000:00000010:0.1:1775773710.839354:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206186a00. 00010000:00000010:0.1:1775773710.839356:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0d00. 00010000:00000010:0.1:1775773710.839357:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206187600. 00010000:00000010:0.1:1775773710.839359:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0e00. 00010000:00000010:0.1:1775773710.839360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb13a00. 00010000:00000010:0.1:1775773710.839362:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0900. 00010000:00000010:0.1:1775773710.839363:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101124400. 00010000:00000010:0.1:1775773710.839365:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd3500. 00010000:00000010:0.1:1775773710.839367:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101125400. 00010000:00000010:0.1:1775773710.839368:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0400. 00010000:00000010:0.1:1775773710.839370:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206186400. 00010000:00000010:0.1:1775773710.839371:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990c00. 00010000:00000010:0.1:1775773710.839374:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72600. 00010000:00000010:0.1:1775773710.839375:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990400. 00010000:00000010:0.1:1775773710.839377:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101124000. 00010000:00000010:0.1:1775773710.839379:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62029e2c00. 00010000:00000010:0.1:1775773710.839397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101125200. 00010000:00000010:0.1:1775773710.839399:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211939f00. 00010000:00000010:0.1:1775773710.839401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101125e00. 00010000:00000010:0.1:1775773710.839403:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211939700. 00010000:00000010:0.1:1775773710.839405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb13800. 00010000:00000010:0.1:1775773710.839406:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990e00. 00010000:00000010:0.1:1775773710.839408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101124a00. 00010000:00000010:0.1:1775773710.839410:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0100. 00010000:00000010:0.1:1775773710.839411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb12c00. 00010000:00000010:0.1:1775773710.839413:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990600. 00010000:00000010:0.1:1775773710.839414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101124600. 00010000:00000010:0.1:1775773710.839416:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c9f0600. 00010000:00000010:0.1:1775773710.839418:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb13600. 00010000:00000010:0.1:1775773710.839420:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211939400. 00010000:00000010:0.1:1775773710.839422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101124800. 00010000:00000010:0.1:1775773710.839424:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211939c00. 00010000:00000010:0.1:1775773710.839425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6101125c00. 00010000:00000010:0.1:1775773710.839427:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211939500. 00010000:00000010:0.1:1775773710.839428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057f8c00. 00010000:00000010:0.1:1775773710.839430:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211939800. 00010000:00000010:0.1:1775773710.839432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335c00. 00010000:00000010:0.1:1775773710.839434:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990800. 00010000:00000010:0.1:1775773710.839435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da72200. 00010000:00000010:0.1:1775773710.839437:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990a00. 00010000:00000010:0.1:1775773710.839439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e334000. 00010000:00000010:0.1:1775773710.839440:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211939300. 00010000:00000010:0.1:1775773710.839442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e334e00. 00010000:00000010:0.1:1775773710.839443:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211939600. 00010000:00000010:0.1:1775773710.839445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e334c00. 00010000:00000010:0.1:1775773710.839447:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211939a00. 00010000:00000010:0.1:1775773710.839449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e334800. 00010000:00000010:0.1:1775773710.839450:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211939000. 00010000:00000010:0.1:1775773710.839452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335400. 00010000:00000010:0.1:1775773710.839454:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620388e300. 00010000:00000010:0.1:1775773710.839455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206187c00. 00010000:00000010:0.1:1775773710.839457:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211939900. 00010000:00000010:0.1:1775773710.839458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206187400. 00010000:00000010:0.1:1775773710.839460:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234990700. 00010000:00000010:0.1:1775773710.839464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e335000. 00000800:00000200:3.1:1775773712.699757:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773712.699759:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775773712.700164:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773712.700166:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775773712.700348:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773712.700350:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775773717.819833:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773717.820180:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773717.820344:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000080:00000010:0.1:1775773721.079183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c6ccd80. 00000080:00000010:0.1:1775773721.079201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c6ccde0. 00000080:00000010:0.1:1775773721.079208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b8e0. 00000080:00000010:0.1:1775773721.079215:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c6cc960. 00000080:00000010:0.1:1775773721.083182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2200. 00000080:00000010:0.1:1775773721.083196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c6ccb20. 00000080:00000010:0.1:1775773721.083204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c6cca40. 00000080:00000010:0.1:1775773721.083213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c6cce40. 00000080:00000010:0.1:1775773721.083219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c6cc2e0. 00000080:00000010:0.1:1775773721.083231:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c6ccfe0. 00000080:00000010:0.1:1775773721.083235:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a24a0. 00000080:00000010:0.1:1775773721.083255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2ba0. 00000080:00000010:0.1:1775773721.083264:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c6cc280. 00000080:00000010:0.1:1775773721.083271:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c6cc240. 00000080:00000010:0.1:1775773721.083285:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c6ccf80. 00000080:00000010:0.1:1775773721.083295:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b940. 00000080:00000010:0.1:1775773721.083306:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c6cc3a0. 00000080:00000010:0.1:1775773721.083315:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2060. 00000080:00000010:0.1:1775773721.083322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c6ccda0. 00000800:00000200:3.1:1775773722.940265:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773722.940274:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775773722.941090:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773722.941093:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775773722.941517:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773722.941520:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000080:00000010:2.1:1775773740.511162:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292c00. 00000080:00000010:2.1:1775773740.511174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233edb500. 00000080:00000010:2.1:1775773740.515178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7ba80. 00000080:00000010:2.1:1775773740.515193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233edb7e0. 00000080:00000010:2.1:1775773740.515198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233edbb80. 00000080:00000010:2.1:1775773740.515203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bde0. 00000080:00000010:2.1:1775773740.515211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292de0. 00000080:00000010:2.1:1775773740.515216:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b340. 00000080:00000010:2.1:1775773740.515221:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292580. 00000080:00000010:2.1:1775773740.515225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bbe0. 00000080:00000010:2.1:1775773740.515234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233edb760. 00000080:00000010:2.1:1775773740.515238:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292c80. 00000080:00000010:2.1:1775773740.515243:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292ca0. 00000080:00000010:2.1:1775773740.515247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b4c0. 00000080:00000010:2.1:1775773740.515253:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd625a0. 00000080:00000010:2.1:1775773740.515261:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c440c40. 00000080:00000010:2.1:1775773740.515264:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c440c00. 00000080:00000010:2.1:1775773740.515268:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bda0. 00000080:00000010:2.1:1775773740.515271:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b9a0. 00000080:00000010:2.1:1775773740.515278:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b7a0. 00000080:00000010:2.1:1775773740.515282:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62102922c0. 00000080:00000010:2.1:1775773740.515286:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b400. 00000080:00000010:2.1:1775773740.515290:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c440600. 00000080:00000010:2.1:1775773740.515293:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c4406c0. 00000080:00000010:2.1:1775773740.515297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0e6c0. 00000080:00000010:2.1:1775773740.515302:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0eba0. 00000080:00000010:2.1:1775773740.515306:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b700. 00000080:00000010:2.1:1775773740.515312:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2fe0. 00000080:00000010:2.1:1775773740.515316:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2960. 00000080:00000010:2.1:1775773740.515320:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2b40. 00000080:00000010:2.1:1775773740.515324:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2120. 00000080:00000010:2.1:1775773740.515328:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292e00. 00000080:00000010:2.1:1775773740.515334:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62102925e0. 00000080:00000010:2.1:1775773740.515340:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292c20. 00000080:00000010:2.1:1775773740.515345:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292260. 00000080:00000010:2.1:1775773740.515349:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62e40. 00000080:00000010:2.1:1775773740.515353:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62c80. 00000080:00000010:2.1:1775773740.515366:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b6e0. 00000080:00000010:2.1:1775773740.515372:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292980. 00000080:00000010:2.1:1775773740.515376:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2140. 00000080:00000010:2.1:1775773740.515380:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292180. 00000080:00000010:2.1:1775773740.515384:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62c60. 00000080:00000010:2.1:1775773740.515388:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a22a0. 00000080:00000010:2.1:1775773740.515392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62e80. 00000080:00000010:2.1:1775773740.515395:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bce0. 00000080:00000010:2.1:1775773740.515399:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292ec0. 00000080:00000010:2.1:1775773740.515403:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292ce0. 00000080:00000010:2.1:1775773740.515407:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b0a0. 00000080:00000010:2.1:1775773740.515411:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62380. 00000080:00000010:2.1:1775773740.515415:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62102921c0. 00000080:00000010:2.1:1775773740.515419:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62d00. 00000080:00000010:2.1:1775773740.515423:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62102929a0. 00000080:00000010:2.1:1775773740.515427:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292fe0. 00000080:00000010:2.1:1775773740.515431:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62f00. 00000080:00000010:2.1:1775773740.515434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62102923c0. 00000080:00000010:2.1:1775773740.515438:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2380. 00000080:00000010:2.1:1775773740.515442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2940. 00000080:00000010:2.1:1775773740.515446:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292b00. 00000080:00000010:2.1:1775773740.515450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292d20. 00000080:00000010:2.1:1775773740.515453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62102922e0. 00000080:00000010:2.1:1775773740.515456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292f80. 00000080:00000010:2.1:1775773740.515460:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62700. 00000080:00000010:2.1:1775773740.515463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7be80. 00000080:00000010:2.1:1775773740.515467:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bfa0. 00000080:00000010:2.1:1775773740.515472:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292220. 00000080:00000010:2.1:1775773740.515477:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2b20. 00000080:00000010:2.1:1775773740.515480:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bb80. 00000080:00000010:2.1:1775773740.515496:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd623a0. 00000080:00000010:2.1:1775773740.515500:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2ea0. 00000080:00000010:2.1:1775773740.515504:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b040. 00000080:00000010:2.1:1775773740.515508:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292880. 00000080:00000010:2.1:1775773740.515513:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292d80. 00000080:00000010:2.1:1775773740.515517:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd627a0. 00000080:00000010:2.1:1775773740.515523:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292540. 00000080:00000010:2.1:1775773740.515527:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292e20. 00000080:00000010:2.1:1775773740.515531:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b0e0. 00000080:00000010:2.1:1775773740.515535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a24e0. 00000080:00000010:2.1:1775773740.515539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2ce0. 00000080:00000010:2.1:1775773740.515543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2400. 00000080:00000010:2.1:1775773740.515560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7ba40. 00000080:00000010:2.1:1775773740.515564:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a26a0. 00000080:00000010:2.1:1775773740.515568:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b260. 00000080:00000010:2.1:1775773740.515571:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62060. 00000080:00000010:2.1:1775773740.515576:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62102924c0. 00000080:00000010:2.1:1775773740.515580:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292ae0. 00000080:00000010:2.1:1775773740.515599:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd628e0. 00000080:00000010:2.1:1775773740.515604:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292020. 00000080:00000010:2.1:1775773740.515609:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62f20. 00000080:00000010:2.1:1775773740.515613:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62120. 00000080:00000010:2.1:1775773740.515617:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2b80. 00000080:00000010:2.1:1775773740.515620:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b2a0. 00000080:00000010:2.1:1775773740.515624:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292120. 00000080:00000010:2.1:1775773740.515627:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292440. 00000080:00000010:2.1:1775773740.515631:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292ac0. 00000080:00000010:2.1:1775773740.515646:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292940. 00000080:00000010:2.1:1775773740.515649:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62660. 00000080:00000010:2.1:1775773740.515653:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2ca0. 00000080:00000010:2.1:1775773740.515657:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2560. 00000080:00000010:2.1:1775773740.515661:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292a80. 00000080:00000010:2.1:1775773740.515665:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bf00. 00000080:00000010:2.1:1775773740.515669:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2d80. 00000080:00000010:2.1:1775773740.515673:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62a60. 00000080:00000010:2.1:1775773740.515677:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62102925a0. 00000080:00000010:2.1:1775773740.515680:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2340. 00000080:00000010:2.1:1775773740.515683:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62f60. 00000080:00000010:2.1:1775773740.515687:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292400. 00000080:00000010:2.1:1775773740.515690:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62102920e0. 00000080:00000010:2.1:1775773740.515693:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292e60. 00000080:00000010:2.1:1775773740.515696:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b7e0. 00010000:00000010:2.1:1775773740.515702:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202e77100. 00010000:00000010:2.1:1775773740.515709:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dc09200. 00010000:00000010:2.1:1775773740.515713:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202e77600. 00010000:00000010:2.1:1775773740.515717:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621079e200. 00010000:00000010:2.1:1775773740.515720:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202e77f00. 00010000:00000010:2.1:1775773740.515724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62108b8c00. 00010000:00000010:2.1:1775773740.515728:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e106400. 00010000:00000010:2.1:1775773740.515731:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621079fe00. 00010000:00000010:2.1:1775773740.515737:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621193af00. 00010000:00000010:2.1:1775773740.515740:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62108b9000. 00010000:00000010:2.1:1775773740.515743:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202e77000. 00010000:00000010:2.1:1775773740.515746:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62108b8400. 00010000:00000010:2.1:1775773740.515748:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e106800. 00010000:00000010:2.1:1775773740.515751:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621079f800. 00010000:00000010:2.1:1775773740.515753:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202e77500. 00010000:00000010:2.1:1775773740.515756:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dc08400. 00010000:00000010:2.1:1775773740.515759:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e106e00. 00010000:00000010:2.1:1775773740.515762:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621079ee00. 00010000:00000010:2.1:1775773740.515764:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202e77b00. 00010000:00000010:2.1:1775773740.515767:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dc08800. 00010000:00000010:2.1:1775773740.515769:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e106900. 00010000:00000010:2.1:1775773740.515772:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621079ea00. 00010000:00000010:2.1:1775773740.515775:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cba6200. 00010000:00000010:2.1:1775773740.515778:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62108b9c00. 00010000:00000010:2.1:1775773740.515782:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210af4200. 00010000:00000010:2.1:1775773740.515786:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dc09800. 00010000:00000010:2.1:1775773740.515789:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210af4700. 00010000:00000010:2.1:1775773740.515792:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dc08600. 00010000:00000010:2.1:1775773740.515795:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cba6d00. 00010000:00000010:2.1:1775773740.515798:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621079fa00. 00010000:00000010:2.1:1775773740.515801:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621193a000. 00010000:00000010:2.1:1775773740.515804:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205728a00. 00010000:00000010:2.1:1775773740.515807:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621193a500. 00010000:00000010:2.1:1775773740.515811:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62108b9e00. 00010000:00000010:2.1:1775773740.515813:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621193ab00. 00010000:00000010:2.1:1775773740.515816:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62108b8200. 00010000:00000010:2.1:1775773740.515818:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210af4a00. 00010000:00000010:2.1:1775773740.515822:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621079f400. 00010000:00000010:2.1:1775773740.515824:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cba6400. 00010000:00000010:2.1:1775773740.515827:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621079ec00. 00010000:00000010:2.1:1775773740.515830:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bc9e300. 00010000:00000010:2.1:1775773740.515835:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c92aa00. 00010000:00000010:2.1:1775773740.515838:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621193ac00. 00010000:00000010:2.1:1775773740.515840:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dc08000. 00010000:00000010:2.1:1775773740.515843:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cba6800. 00010000:00000010:2.1:1775773740.515847:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c92a000. 00010000:00000010:2.1:1775773740.515849:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210af4100. 00010000:00000010:2.1:1775773740.515852:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62108b8600. 00010000:00000010:2.1:1775773740.515855:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621193a300. 00010000:00000010:2.1:1775773740.515858:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62108b9800. 00010000:00000010:2.1:1775773740.515861:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210af4600. 00010000:00000010:2.1:1775773740.515863:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62108b8000. 00010000:00000010:2.1:1775773740.515866:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210af4f00. 00010000:00000010:2.1:1775773740.515868:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62108b8a00. 00010000:00000010:2.1:1775773740.515871:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621193a200. 00010000:00000010:2.1:1775773740.515875:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c92b400. 00010000:00000010:2.1:1775773740.515878:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621193ad00. 00010000:00000010:2.1:1775773740.515880:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62108b8e00. 00010000:00000010:2.1:1775773740.515883:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bc9e200. 00010000:00000010:2.1:1775773740.515886:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62108b9a00. 00010000:00000010:2.1:1775773740.515889:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bc9ed00. 00010000:00000010:2.1:1775773740.515893:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62108b8800. 00010000:00000010:2.1:1775773740.515896:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bc9e400. 00010000:00000010:2.1:1775773740.515900:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233756c00. 00010000:00000010:2.1:1775773740.515903:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210af4000. 00010000:00000010:2.1:1775773740.515906:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dc08a00. 00010000:00000010:2.1:1775773740.515909:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bc9e800. 00010000:00000010:2.1:1775773740.515912:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dc09400. 00010000:00000010:2.1:1775773740.515915:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210af4500. 00010000:00000010:2.1:1775773740.515918:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dc09c00. 00010000:00000010:2.1:1775773740.515921:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210af4b00. 00010000:00000010:2.1:1775773740.515923:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dc08c00. 00010000:00000010:2.1:1775773740.515927:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210af4c00. 00010000:00000010:2.1:1775773740.515930:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205729400. 00010000:00000010:2.1:1775773740.515933:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621193a400. 00010000:00000010:2.1:1775773740.515936:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205729c00. 00010000:00000010:2.1:1775773740.515939:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cba6e00. 00010000:00000010:2.1:1775773740.515942:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c92a800. 00010000:00000010:2.1:1775773740.515945:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210af4300. 00010000:00000010:2.1:1775773740.515949:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234acf600. 00010000:00000010:2.1:1775773740.515954:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cba6900. 00010000:00000010:2.1:1775773740.515956:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233756000. 00010000:00000010:2.1:1775773740.515959:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cba6700. 00010000:00000010:2.1:1775773740.515962:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234acf000. 00010000:00000010:2.1:1775773740.515965:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621193a800. 00010000:00000010:2.1:1775773740.515967:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205728c00. 00010000:00000010:2.1:1775773740.515970:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621193ae00. 00010000:00000010:2.1:1775773740.515974:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233757600. 00010000:00000010:2.1:1775773740.515977:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621193a900. 00010000:00000010:2.1:1775773740.515980:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205729600. 00010000:00000010:2.1:1775773740.515983:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210af4d00. 00010000:00000010:2.1:1775773740.515986:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c92a200. 00010000:00000010:2.1:1775773740.515988:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210af4400. 00010000:00000010:2.1:1775773740.515991:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234acfe00. 00010000:00000010:2.1:1775773740.515994:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210af4800. 00010000:00000010:2.1:1775773740.515997:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234acf200. 00010000:00000010:2.1:1775773740.516000:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cba6a00. 00010000:00000010:2.1:1775773740.516003:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c92b000. 00010000:00000010:2.1:1775773740.516006:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621193a700. 00010000:00000010:2.1:1775773740.516010:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e7e5600. 00010000:00000010:2.1:1775773740.516013:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210af4e00. 00010000:00000010:2.1:1775773740.516016:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234ace400. 00010000:00000010:2.1:1775773740.516018:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621193aa00. 00010000:00000010:2.1:1775773740.516023:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e7e4e00. 00010000:00000010:2.1:1775773740.516026:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210af4900. 00010000:00000010:2.1:1775773740.516029:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234ace800. 00010000:00000010:2.1:1775773740.516034:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd61400. 00010000:00000010:2.1:1775773740.516038:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234acf800. 00010000:00000010:2.1:1775773740.516040:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621193a100. 00010000:00000010:2.1:1775773740.516043:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e7e4200. 00010000:00000010:2.1:1775773740.516046:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd61800. 00010000:00000010:2.1:1775773740.516049:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234ace600. 00010000:00000010:2.1:1775773740.516052:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd61e00. 00010000:00000010:2.1:1775773740.516055:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233757e00. 00010000:00000010:2.1:1775773740.516058:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bc9ee00. 00010000:00000010:2.1:1775773740.516061:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233757800. 00010000:00000010:2.1:1775773740.516063:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd61900. 00010000:00000010:2.1:1775773740.516066:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234ace000. 00010000:00000010:2.1:1775773740.516069:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd61700. 00010000:00000010:2.1:1775773740.516071:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234acea00. 00010000:00000010:2.1:1775773740.516074:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd61a00. 00010000:00000010:2.1:1775773740.516077:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234acf400. 00010000:00000010:2.1:1775773740.516080:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd61100. 00010000:00000010:2.1:1775773740.516083:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234acfc00. 00010000:00000010:2.1:1775773740.516088:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dce8100. 00010000:00000010:2.1:1775773740.516092:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e7e5a00. 00010000:00000010:2.1:1775773740.516096:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207da4c00. 00010000:00000010:2.1:1775773740.516099:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c92ba00. 00010000:00000010:2.1:1775773740.516101:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207da4200. 00010000:00000010:2.1:1775773740.516104:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c92b600. 00010000:00000010:2.1:1775773740.516107:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd61600. 00010000:00000010:2.1:1775773740.516110:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234acec00. 00010000:00000010:2.1:1775773740.516113:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd61f00. 00010000:00000010:2.1:1775773740.516116:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233756a00. 00010000:00000010:2.1:1775773740.516119:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cba6100. 00010000:00000010:2.1:1775773740.516122:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c92a400. 00010000:00000010:2.1:1775773740.516125:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dce8600. 00010000:00000010:2.1:1775773740.516128:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e7e5000. 00010000:00000010:2.1:1775773740.516131:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd61000. 00010000:00000010:2.1:1775773740.516152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233756400. 00010000:00000010:2.1:1775773740.516156:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cba6600. 00010000:00000010:2.1:1775773740.516158:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c92ac00. 00010000:00000010:2.1:1775773740.516161:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd61500. 00010000:00000010:2.1:1775773740.516163:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234acee00. 00010000:00000010:2.1:1775773740.516166:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd61b00. 00010000:00000010:2.1:1775773740.516168:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234ace200. 00010000:00000010:2.1:1775773740.516171:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dce8f00. 00010000:00000010:2.1:1775773740.516174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e7e5e00. 00010000:00000010:2.1:1775773740.516177:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd61c00. 00010000:00000010:2.1:1775773740.516180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234acfa00. 00010000:00000010:2.1:1775773740.516183:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd61300. 00010000:00000010:2.1:1775773740.516187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e568a00. 00010000:00000010:2.1:1775773740.516190:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd61200. 00010000:00000010:2.1:1775773740.516193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c92be00. 00010000:00000010:2.1:1775773740.516196:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207da4400. 00010000:00000010:2.1:1775773740.516200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233757c00. 00010000:00000010:2.1:1775773740.516203:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207da4700. 00010000:00000010:2.1:1775773740.516206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233756600. 00010000:00000010:2.1:1775773740.516209:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207da4500. 00010000:00000010:2.1:1775773740.516212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233756200. 00010000:00000010:2.1:1775773740.516215:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd61d00. 00010000:00000010:2.1:1775773740.516218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c92ae00. 00010000:00000010:2.1:1775773740.516221:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207da4f00. 00010000:00000010:2.1:1775773740.516225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233757200. 00010000:00000010:2.1:1775773740.516228:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207da4e00. 00010000:00000010:2.1:1775773740.516231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c92a600. 00010000:00000010:2.1:1775773740.516234:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dce8000. 00010000:00000010:2.1:1775773740.516237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e7e5200. 00010000:00000010:2.1:1775773740.516241:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6201325600. 00010000:00000010:2.1:1775773740.516245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e569400. 00010000:00000010:2.1:1775773740.516247:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6201325f00. 00010000:00000010:2.1:1775773740.516250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e569c00. 00010000:00000010:2.1:1775773740.516253:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dce8500. 00010000:00000010:2.1:1775773740.516256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e7e4400. 00010000:00000010:2.1:1775773740.516259:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6201325000. 00010000:00000010:2.1:1775773740.516262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e568c00. 00010000:00000010:2.1:1775773740.516265:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dce8b00. 00010000:00000010:2.1:1775773740.516267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e7e4800. 00010000:00000010:2.1:1775773740.516270:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dce8c00. 00010000:00000010:2.1:1775773740.516272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e7e5800. 00010000:00000010:2.1:1775773740.516275:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207da4100. 00010000:00000010:2.1:1775773740.516278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233756e00. 00010000:00000010:2.1:1775773740.516281:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207da4300. 00010000:00000010:2.1:1775773740.516283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c92bc00. 00010000:00000010:2.1:1775773740.516286:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6201325500. 00010000:00000010:2.1:1775773740.516289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e569600. 00010000:00000010:2.1:1775773740.516292:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6201325b00. 00010000:00000010:2.1:1775773740.516295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e568e00. 00010000:00000010:2.1:1775773740.516298:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6201325c00. 00010000:00000010:2.1:1775773740.516301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e568200. 00010000:00000010:2.1:1775773740.516304:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6201325300. 00010000:00000010:2.1:1775773740.516307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e569a00. 00010000:00000010:2.1:1775773740.516310:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dce8300. 00010000:00000010:2.1:1775773740.516314:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e7e4600. 00010000:00000010:2.1:1775773740.516317:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207da4d00. 00010000:00000010:2.1:1775773740.516320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233756800. 00010000:00000010:2.1:1775773740.516324:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207da4000. 00010000:00000010:2.1:1775773740.516327:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233757a00. 00010000:00000010:2.1:1775773740.516330:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6201325200. 00010000:00000010:2.1:1775773740.516333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e569000. 00010000:00000010:2.1:1775773740.516337:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6201325d00. 00010000:00000010:2.1:1775773740.516339:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c92b800. 00010000:00000010:2.1:1775773740.516342:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dce8200. 00010000:00000010:2.1:1775773740.516345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233757000. 00010000:00000010:2.1:1775773740.516348:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dce8d00. 00010000:00000010:2.1:1775773740.516352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e7e4000. 00010000:00000010:2.1:1775773740.516354:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6201325400. 00010000:00000010:2.1:1775773740.516358:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e569e00. 00010000:00000010:2.1:1775773740.516361:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dce8400. 00010000:00000010:2.1:1775773740.516364:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233757400. 00010000:00000010:2.1:1775773740.516367:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6201325800. 00010000:00000010:2.1:1775773740.516370:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e7e4a00. 00010000:00000010:2.1:1775773740.516373:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6201325e00. 00010000:00000010:2.1:1775773740.516377:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e569200. 00010000:00000010:2.1:1775773740.516379:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6201325900. 00010000:00000010:2.1:1775773740.516383:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e568400. 00010000:00000010:2.1:1775773740.516385:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dce8800. 00010000:00000010:2.1:1775773740.516388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e568800. 00000080:00000010:2.1:1775773750.782182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93ce0. 00000080:00000010:2.1:1775773750.782198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f9e0. 00000080:00000010:2.1:1775773750.786181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93a60. 00000080:00000010:2.1:1775773750.786190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf933a0. 00000080:00000010:2.1:1775773750.786193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf934c0. 00000080:00000010:2.1:1775773750.786198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93360. 00000080:00000010:2.1:1775773750.786202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf933c0. 00000080:00000010:2.1:1775773750.786206:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db47a0. 00000080:00000010:2.1:1775773750.786213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eba60. 00000080:00000010:2.1:1775773750.786217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4620. 00000080:00000010:2.1:1775773750.786220:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf939a0. 00000080:00000010:2.1:1775773750.786227:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93280. 00000080:00000010:2.1:1775773750.786230:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4da0. 00000080:00000010:2.1:1775773750.786232:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93ea0. 00000080:00000010:2.1:1775773750.786244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4300. 00000080:00000010:2.1:1775773750.786247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93220. 00000080:00000010:2.1:1775773750.786249:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93f40. 00000080:00000010:2.1:1775773750.786252:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233edbf20. 00000080:00000010:2.1:1775773750.786255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233edb4c0. 00000080:00000010:2.1:1775773750.786260:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db43e0. 00000080:00000010:2.1:1775773750.786262:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4080. 00000080:00000010:2.1:1775773750.786265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233edb340. 00000080:00000010:2.1:1775773750.786267:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93620. 00000080:00000010:2.1:1775773750.786270:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb920. 00000080:00000010:2.1:1775773750.786273:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93140. 00000080:00000010:2.1:1775773750.786276:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93380. 00000080:00000010:2.1:1775773750.786278:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4e00. 00000080:00000010:2.1:1775773750.786282:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4560. 00000080:00000010:2.1:1775773750.786288:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb480. 00000080:00000010:2.1:1775773750.786293:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4c80. 00000080:00000010:2.1:1775773750.786298:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62220. 00000080:00000010:2.1:1775773750.786302:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93fa0. 00000080:00000010:2.1:1775773750.786306:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db46c0. 00000080:00000010:2.1:1775773750.786311:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd12ca0. 00000080:00000010:2.1:1775773750.786317:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93b60. 00000080:00000010:2.1:1775773750.786321:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93180. 00000080:00000010:2.1:1775773750.786327:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db43c0. 00000080:00000010:2.1:1775773750.786331:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93000. 00000080:00000010:2.1:1775773750.786336:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620aae2240. 00000080:00000010:2.1:1775773750.786341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2740. 00000080:00000010:2.1:1775773750.786344:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93120. 00000080:00000010:2.1:1775773750.786349:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792c3e0. 00000080:00000010:2.1:1775773750.786351:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62ac0. 00000080:00000010:2.1:1775773750.786354:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb8a0. 00000080:00000010:2.1:1775773750.786357:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792c520. 00000080:00000010:2.1:1775773750.786360:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057ebb40. 00000080:00000010:2.1:1775773750.786362:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb240. 00000080:00000010:2.1:1775773750.786366:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93100. 00000080:00000010:2.1:1775773750.786369:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93dc0. 00000080:00000010:2.1:1775773750.786371:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93de0. 00000080:00000010:2.1:1775773750.786376:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf930e0. 00000080:00000010:2.1:1775773750.786378:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93800. 00000080:00000010:2.1:1775773750.786381:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf937a0. 00000080:00000010:2.1:1775773750.786384:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93a80. 00000080:00000010:2.1:1775773750.786386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd12c00. 00000080:00000010:2.1:1775773750.786390:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf931e0. 00000080:00000010:2.1:1775773750.786392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf938a0. 00000080:00000010:2.1:1775773750.786395:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93fc0. 00000080:00000010:2.1:1775773750.786397:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93e60. 00000080:00000010:2.1:1775773750.786399:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93460. 00000080:00000010:2.1:1775773750.786401:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb500. 00000080:00000010:2.1:1775773750.786418:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93660. 00000080:00000010:2.1:1775773750.786424:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792c540. 00000080:00000010:2.1:1775773750.786426:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93e40. 00000080:00000010:2.1:1775773750.786428:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93e80. 00000080:00000010:2.1:1775773750.786431:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93d00. 00000080:00000010:2.1:1775773750.786434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93760. 00000080:00000010:2.1:1775773750.786436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93440. 00000080:00000010:2.1:1775773750.786439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb960. 00000080:00000010:2.1:1775773750.786442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb860. 00000080:00000010:2.1:1775773750.786445:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792c960. 00000080:00000010:2.1:1775773750.786447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792c320. 00000080:00000010:2.1:1775773750.786450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93240. 00000080:00000010:2.1:1775773750.786452:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792cb20. 00000080:00000010:2.1:1775773750.786455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792c040. 00000080:00000010:2.1:1775773750.786458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792c9a0. 00000080:00000010:2.1:1775773750.786461:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792cbc0. 00000080:00000010:2.1:1775773750.786463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93aa0. 00000080:00000010:2.1:1775773750.786467:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf936c0. 00000080:00000010:2.1:1775773750.786469:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93ba0. 00000080:00000010:2.1:1775773750.786472:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93ac0. 00000080:00000010:2.1:1775773750.786475:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf937e0. 00000080:00000010:2.1:1775773750.786477:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93d60. 00000080:00000010:2.1:1775773750.786480:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93040. 00000080:00000010:2.1:1775773750.786482:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf936a0. 00000080:00000010:2.1:1775773750.786484:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93d20. 00000080:00000010:2.1:1775773750.786486:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93900. 00000080:00000010:2.1:1775773750.786488:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93a00. 00000080:00000010:2.1:1775773750.786490:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2580. 00000080:00000010:2.1:1775773750.786492:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a21e0. 00000080:00000010:2.1:1775773750.786494:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792c4c0. 00000080:00000010:2.1:1775773750.786497:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd123a0. 00000080:00000010:2.1:1775773750.786499:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792cce0. 00000080:00000010:2.1:1775773750.786501:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792ca80. 00000080:00000010:2.1:1775773750.786504:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb600. 00000080:00000010:2.1:1775773750.786506:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb9a0. 00000080:00000010:2.1:1775773750.786509:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb680. 00000080:00000010:2.1:1775773750.786511:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c440140. 00000080:00000010:2.1:1775773750.786516:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057ebdc0. 00000080:00000010:2.1:1775773750.786518:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c440bc0. 00000080:00000010:2.1:1775773750.786521:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bb00. 00000080:00000010:2.1:1775773750.786523:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c440b60. 00000080:00000010:2.1:1775773750.786525:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd121c0. 00000080:00000010:2.1:1775773750.786528:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620ac0efa0. 00000080:00000010:2.1:1775773750.786530:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c440b40. 00000080:00000010:2.1:1775773750.786532:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c440680. 00000080:00000010:2.1:1775773750.786535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb900. 00000080:00000010:2.1:1775773750.786537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792c220. 00000080:00000010:2.1:1775773750.786539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792c9c0. 00000080:00000010:2.1:1775773750.786544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292720. 00000080:00000010:2.1:1775773750.786547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7ba60. 00000080:00000010:2.1:1775773750.786549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620aae2900. 00000080:00000010:2.1:1775773750.786552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620aae2b60. 00000080:00000010:2.1:1775773750.786554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62102927a0. 00000080:00000010:2.1:1775773750.786557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bca0. 00000080:00000010:2.1:1775773750.786560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292620. 00000080:00000010:2.1:1775773750.786563:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292680. 00000080:00000010:2.1:1775773750.786565:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292b80. 00000080:00000010:2.1:1775773750.786568:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292040. 00000080:00000010:2.1:1775773750.786570:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd627e0. 00000080:00000010:2.1:1775773750.786573:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057ebe80. 00000080:00000010:2.1:1775773750.786576:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb280. 00000080:00000010:2.1:1775773750.786579:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292d60. 00000080:00000010:2.1:1775773750.786582:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb0a0. 00000080:00000010:2.1:1775773750.786585:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62320. 00000080:00000010:2.1:1775773750.786587:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292200. 00000080:00000010:2.1:1775773750.786589:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057ebc40. 00000080:00000010:2.1:1775773750.786592:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792cc40. 00000080:00000010:2.1:1775773750.786594:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292f40. 00000080:00000010:2.1:1775773750.786597:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd622c0. 00000080:00000010:2.1:1775773750.786599:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62800. 00000080:00000010:2.1:1775773750.786602:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62102923a0. 00000080:00000010:2.1:1775773750.786605:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292080. 00000080:00000010:2.1:1775773750.786607:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292920. 00000080:00000010:2.1:1775773750.786610:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292f60. 00000080:00000010:2.1:1775773750.786613:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292c40. 00000080:00000010:2.1:1775773750.786615:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292840. 00000080:00000010:2.1:1775773750.786618:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62102921a0. 00000080:00000010:2.1:1775773750.786620:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292420. 00000080:00000010:2.1:1775773750.786623:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292480. 00000080:00000010:2.1:1775773750.786625:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb4c0. 00000080:00000010:2.1:1775773750.786628:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62102926c0. 00000080:00000010:2.1:1775773750.786630:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057ebb20. 00000080:00000010:2.1:1775773750.786632:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb3c0. 00000080:00000010:2.1:1775773750.786635:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd623e0. 00000080:00000010:2.1:1775773750.786638:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233edb180. 00000080:00000010:2.1:1775773750.786640:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62780. 00000080:00000010:2.1:1775773750.786642:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792cde0. 00000080:00000010:2.1:1775773750.786645:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62102920c0. 00000080:00000010:2.1:1775773750.786647:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792c6c0. 00000080:00000010:2.1:1775773750.786649:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792c3a0. 00000080:00000010:2.1:1775773750.786651:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233edbe00. 00000080:00000010:2.1:1775773750.786654:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292a00. 00000080:00000010:2.1:1775773750.786657:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62102927e0. 00000080:00000010:2.1:1775773750.786659:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292660. 00000080:00000010:2.1:1775773750.786662:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292460. 00000080:00000010:2.1:1775773750.786665:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292b40. 00000080:00000010:2.1:1775773750.786669:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62102920a0. 00000080:00000010:2.1:1775773750.786671:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62b20. 00000080:00000010:2.1:1775773750.786673:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62540. 00000080:00000010:2.1:1775773750.786675:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292d40. 00000080:00000010:2.1:1775773750.786678:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792cec0. 00000080:00000010:2.1:1775773750.786680:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233edb460. 00000080:00000010:2.1:1775773750.786682:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292ea0. 00000080:00000010:2.1:1775773750.786684:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792c900. 00000080:00000010:2.1:1775773750.786687:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292fc0. 00000080:00000010:2.1:1775773750.786689:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292700. 00000080:00000010:2.1:1775773750.786691:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62c00. 00000080:00000010:2.1:1775773750.786693:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62d80. 00000080:00000010:2.1:1775773750.786695:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292da0. 00000080:00000010:2.1:1775773750.786698:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792c760. 00000080:00000010:2.1:1775773750.786700:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292640. 00000080:00000010:2.1:1775773750.786702:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292280. 00000080:00000010:2.1:1775773750.786704:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292240. 00000080:00000010:2.1:1775773750.786706:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233edb2e0. 00000080:00000010:2.1:1775773750.786709:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bb20. 00000080:00000010:2.1:1775773750.786711:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292cc0. 00000080:00000010:2.1:1775773750.786713:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62102928a0. 00000080:00000010:2.1:1775773750.786715:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292c60. 00000080:00000010:2.1:1775773750.786718:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bd60. 00000080:00000010:2.1:1775773750.786720:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b960. 00000080:00000010:2.1:1775773750.786721:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b080. 00000080:00000010:2.1:1775773750.786723:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292760. 00000080:00000010:2.1:1775773750.786725:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b140. 00000080:00000010:2.1:1775773750.786727:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292ba0. 00000080:00000010:2.1:1775773750.786729:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7bee0. 00000080:00000010:2.1:1775773750.786731:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cd62e60. 00000080:00000010:2.1:1775773750.786733:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292800. 00000080:00000010:2.1:1775773750.786736:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b000. 00000080:00000010:2.1:1775773750.786738:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233edb440. 00000080:00000010:2.1:1775773750.786741:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210292360. 00010000:00000010:2.1:1775773750.786745:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ccecb00. 00010000:00000010:2.1:1775773750.786749:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e62ac00. 00010000:00000010:2.1:1775773750.786752:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ccecc00. 00010000:00000010:2.1:1775773750.786754:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234999800. 00010000:00000010:2.1:1775773750.786756:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ccec300. 00010000:00000010:2.1:1775773750.786759:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e62b600. 00010000:00000010:2.1:1775773750.786760:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ccec200. 00010000:00000010:2.1:1775773750.786762:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e62ae00. 00010000:00000010:2.1:1775773750.786764:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ccecd00. 00010000:00000010:2.1:1775773750.786766:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e62a200. 00010000:00000010:2.1:1775773750.786768:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ccec400. 00010000:00000010:2.1:1775773750.786769:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e62ba00. 00010000:00000010:2.1:1775773750.786771:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ccec800. 00010000:00000010:2.1:1775773750.786773:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e62b000. 00010000:00000010:2.1:1775773750.786775:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234947200. 00010000:00000010:2.1:1775773750.786777:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c707c00. 00010000:00000010:2.1:1775773750.786779:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bc9ef00. 00010000:00000010:2.1:1775773750.786781:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234998c00. 00010000:00000010:2.1:1775773750.786783:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205044b00. 00010000:00000010:2.1:1775773750.786786:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb16800. 00010000:00000010:2.1:1775773750.786788:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234947d00. 00010000:00000010:2.1:1775773750.786790:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e62be00. 00010000:00000010:2.1:1775773750.786791:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234947400. 00010000:00000010:2.1:1775773750.786793:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e62b200. 00010000:00000010:2.1:1775773750.786795:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bc9e000. 00010000:00000010:2.1:1775773750.786797:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb17c00. 00010000:00000010:2.1:1775773750.786799:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234947800. 00010000:00000010:2.1:1775773750.786801:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e62a400. 00010000:00000010:2.1:1775773750.786802:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620adff500. 00010000:00000010:2.1:1775773750.786805:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb17600. 00010000:00000010:2.1:1775773750.786807:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205044c00. 00010000:00000010:2.1:1775773750.786808:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e62a800. 00010000:00000010:2.1:1775773750.786810:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234947e00. 00010000:00000010:2.1:1775773750.786812:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e62b800. 00010000:00000010:2.1:1775773750.786814:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234947900. 00010000:00000010:2.1:1775773750.786816:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234999000. 00010000:00000010:2.1:1775773750.786817:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205044300. 00010000:00000010:2.1:1775773750.786819:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234999e00. 00010000:00000010:2.1:1775773750.786820:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234947700. 00010000:00000010:2.1:1775773750.786822:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb16400. 00010000:00000010:2.1:1775773750.786824:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620adffb00. 00010000:00000010:2.1:1775773750.786825:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb17200. 00010000:00000010:2.1:1775773750.786827:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234947a00. 00010000:00000010:2.1:1775773750.786828:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234999200. 00010000:00000010:2.1:1775773750.786830:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620adffc00. 00010000:00000010:2.1:1775773750.786831:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e62a600. 00010000:00000010:2.1:1775773750.786833:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620adff300. 00010000:00000010:2.1:1775773750.786835:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234998a00. 00010000:00000010:2.1:1775773750.786836:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234947100. 00010000:00000010:2.1:1775773750.786838:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e62a000. 00010000:00000010:2.1:1775773750.786840:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234947600. 00010000:00000010:2.1:1775773750.786841:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e62aa00. 00010000:00000010:2.1:1775773750.786843:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205044200. 00010000:00000010:2.1:1775773750.786845:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb17000. 00010000:00000010:2.1:1775773750.786846:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205044d00. 00010000:00000010:2.1:1775773750.786848:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb16200. 00010000:00000010:2.1:1775773750.786850:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233ae2900. 00010000:00000010:2.1:1775773750.786852:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234998e00. 00010000:00000010:2.1:1775773750.786854:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233ae2700. 00010000:00000010:2.1:1775773750.786856:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb17a00. 00010000:00000010:2.1:1775773750.786858:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233ae2a00. 00010000:00000010:2.1:1775773750.786860:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234999a00. 00010000:00000010:2.1:1775773750.786862:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234947f00. 00010000:00000010:2.1:1775773750.786864:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220673400. 00010000:00000010:2.1:1775773750.786866:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234947000. 00010000:00000010:2.1:1775773750.786868:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb17e00. 00010000:00000010:2.1:1775773750.786870:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233ae2100. 00010000:00000010:2.1:1775773750.786872:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62066dba00. 00010000:00000010:2.1:1775773750.786874:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234947500. 00010000:00000010:2.1:1775773750.786875:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220673c00. 00010000:00000010:2.1:1775773750.786877:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233ae2600. 00010000:00000010:2.1:1775773750.786879:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62066db000. 00010000:00000010:2.1:1775773750.786880:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233ae2f00. 00010000:00000010:2.1:1775773750.786882:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb17800. 00010000:00000010:2.1:1775773750.786883:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234947b00. 00010000:00000010:2.1:1775773750.786886:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220672c00. 00010000:00000010:2.1:1775773750.786887:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234947c00. 00010000:00000010:2.1:1775773750.786889:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb16600. 00010000:00000010:2.1:1775773750.786891:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234947300. 00010000:00000010:2.1:1775773750.786892:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb16a00. 00010000:00000010:2.1:1775773750.786895:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c51dc00. 00010000:00000010:2.1:1775773750.786897:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220673600. 00010000:00000010:2.1:1775773750.786899:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c51d300. 00010000:00000010:2.1:1775773750.786900:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb17400. 00010000:00000010:2.1:1775773750.786902:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233ae2000. 00010000:00000010:2.1:1775773750.786903:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234999400. 00010000:00000010:2.1:1775773750.786905:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c51d200. 00010000:00000010:2.1:1775773750.786907:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234999c00. 00010000:00000010:2.1:1775773750.786908:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233ae2500. 00010000:00000010:2.1:1775773750.786910:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb16e00. 00010000:00000010:2.1:1775773750.786912:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205044400. 00010000:00000010:2.1:1775773750.786914:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234999600. 00010000:00000010:2.1:1775773750.786915:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205044800. 00010000:00000010:2.1:1775773750.786917:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234998200. 00010000:00000010:2.1:1775773750.786919:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c51dd00. 00010000:00000010:2.1:1775773750.786921:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220672e00. 00010000:00000010:2.1:1775773750.786923:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c51d400. 00010000:00000010:2.1:1775773750.786925:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220672200. 00010000:00000010:2.1:1775773750.786926:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c51d800. 00010000:00000010:2.1:1775773750.786928:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220673a00. 00010000:00000010:2.1:1775773750.786930:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233ae2b00. 00010000:00000010:2.1:1775773750.786931:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220673000. 00010000:00000010:2.1:1775773750.786933:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c51de00. 00010000:00000010:2.1:1775773750.786935:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220673e00. 00010000:00000010:2.1:1775773750.786936:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c51d900. 00010000:00000010:2.1:1775773750.786938:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220673200. 00010000:00000010:2.1:1775773750.786940:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c51d700. 00010000:00000010:2.1:1775773750.786941:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220672400. 00010000:00000010:2.1:1775773750.786943:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c51da00. 00010000:00000010:2.1:1775773750.786945:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62066dbe00. 00010000:00000010:2.1:1775773750.786946:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c51d100. 00010000:00000010:2.1:1775773750.786948:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220672800. 00010000:00000010:2.1:1775773750.786949:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c51d600. 00010000:00000010:2.1:1775773750.786951:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220673800. 00010000:00000010:2.1:1775773750.786952:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c51df00. 00010000:00000010:2.1:1775773750.786954:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220672600. 00010000:00000010:2.1:1775773750.786956:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c51d000. 00010000:00000010:2.1:1775773750.786957:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220672000. 00010000:00000010:2.1:1775773750.786959:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c51d500. 00010000:00000010:2.1:1775773750.786960:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6220672a00. 00010000:00000010:2.1:1775773750.786962:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bc9e500. 00010000:00000010:2.1:1775773750.786964:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234998800. 00010000:00000010:2.1:1775773750.786965:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c51db00. 00010000:00000010:2.1:1775773750.786967:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62063c9800. 00010000:00000010:2.1:1775773750.786970:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205766b00. 00010000:00000010:2.1:1775773750.786972:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb16c00. 00010000:00000010:2.1:1775773750.786974:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205766c00. 00010000:00000010:2.1:1775773750.786976:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62063c8600. 00010000:00000010:2.1:1775773750.786978:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205766300. 00010000:00000010:2.1:1775773750.786980:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62063c8000. 00010000:00000010:2.1:1775773750.786982:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205766200. 00010000:00000010:2.1:1775773750.786984:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62063c8a00. 00010000:00000010:2.1:1775773750.786985:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205766d00. 00010000:00000010:2.1:1775773750.786987:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62063c9400. 00010000:00000010:2.1:1775773750.786989:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205766400. 00010000:00000010:2.1:1775773750.786991:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62063c9c00. 00010000:00000010:2.1:1775773750.786992:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205044e00. 00010000:00000010:2.1:1775773750.786994:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e62ae00. 00010000:00000010:2.1:1775773750.786996:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205766800. 00010000:00000010:2.1:1775773750.786999:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e62b600. 00010000:00000010:2.1:1775773750.787001:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621ddaf100. 00010000:00000010:2.1:1775773750.787003:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb16000. 00010000:00000010:2.1:1775773750.787005:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621ddaff00. 00010000:00000010:2.1:1775773750.787007:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62067b2200. 00010000:00000010:2.1:1775773750.787028:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233ae2c00. 00010000:00000010:2.1:1775773750.787030:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62063c8c00. 00010000:00000010:2.1:1775773750.787033:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621ddaf000. 00010000:00000010:2.1:1775773750.787035:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62067b3a00. 00010000:00000010:2.1:1775773750.787036:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621ddaf500. 00010000:00000010:2.1:1775773750.787038:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62067b3000. 00010000:00000010:2.1:1775773750.787040:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621ddafb00. 00010000:00000010:2.1:1775773750.787041:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62067b3800. 00010000:00000010:2.1:1775773750.787043:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205766e00. 00010000:00000010:2.1:1775773750.787045:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62067b3400. 00010000:00000010:2.1:1775773750.787047:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205766900. 00010000:00000010:2.1:1775773750.787049:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62063c9600. 00010000:00000010:2.1:1775773750.787051:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205766700. 00010000:00000010:2.1:1775773750.787053:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62063c8e00. 00010000:00000010:2.1:1775773750.787055:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205766a00. 00010000:00000010:2.1:1775773750.787057:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62063c8200. 00010000:00000010:2.1:1775773750.787059:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205766100. 00010000:00000010:2.1:1775773750.787061:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62063c9a00. 00010000:00000010:2.1:1775773750.787062:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205766600. 00010000:00000010:2.1:1775773750.787064:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62063c9000. 00010000:00000010:2.1:1775773750.787066:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205766f00. 00010000:00000010:2.1:1775773750.787067:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62063c9e00. 00010000:00000010:2.1:1775773750.787069:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205766000. 00010000:00000010:2.1:1775773750.787071:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62063c9200. 00010000:00000010:2.1:1775773750.787072:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205766500. 00010000:00000010:2.1:1775773750.787074:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62063c8400. 00010000:00000010:2.1:1775773750.787076:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62052b3b00. 00010000:00000010:2.1:1775773750.787090:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62063c8800. 00010000:00000010:2.1:1775773750.787095:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62052b3c00. 00010000:00000010:2.1:1775773750.787097:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bee000. 00010000:00000010:2.1:1775773750.787101:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62052b3300. 00010000:00000010:2.1:1775773750.787102:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233beea00. 00010000:00000010:2.1:1775773750.787105:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62052b3200. 00010000:00000010:2.1:1775773750.787106:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bef400. 00010000:00000010:2.1:1775773750.787110:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62052b3d00. 00010000:00000010:2.1:1775773750.787111:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233befc00. 00010000:00000010:2.1:1775773750.787113:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621ddafc00. 00010000:00000010:2.1:1775773750.787115:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62067b3c00. 00010000:00000010:2.1:1775773750.787116:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233ae2300. 00010000:00000010:2.1:1775773750.787118:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62066da400. 00010000:00000010:2.1:1775773750.787120:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621ddaf300. 00010000:00000010:2.1:1775773750.787122:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62067b2e00. 00010000:00000010:2.1:1775773750.787124:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621ddaf200. 00010000:00000010:2.1:1775773750.787126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62067b2000. 00010000:00000010:2.1:1775773750.787128:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62052b3400. 00010000:00000010:2.1:1775773750.787130:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e62bc00. 00010000:00000010:2.1:1775773750.787146:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62052b3800. 00010000:00000010:2.1:1775773750.787149:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e62b400. 00010000:00000010:2.1:1775773750.787150:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621ddafd00. 00010000:00000010:2.1:1775773750.787152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e62b800. 00010000:00000010:2.1:1775773750.787154:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62052b3e00. 00010000:00000010:2.1:1775773750.787156:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233beec00. 00010000:00000010:2.1:1775773750.787158:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bc9eb00. 00010000:00000010:2.1:1775773750.787160:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e62aa00. 00010000:00000010:2.1:1775773750.787162:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233ae2200. 00010000:00000010:2.1:1775773750.787163:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bef600. 00010000:00000010:2.1:1775773750.787165:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62052b3900. 00010000:00000010:2.1:1775773750.787167:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233beee00. 00010000:00000010:2.1:1775773750.787168:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621ddaf400. 00010000:00000010:2.1:1775773750.787170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bee200. 00010000:00000010:2.1:1775773750.787172:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233ae2d00. 00010000:00000010:2.1:1775773750.787173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62066db200. 00010000:00000010:2.1:1775773750.787175:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621ddaf800. 00010000:00000010:2.1:1775773750.787177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233befa00. 00010000:00000010:2.1:1775773750.787178:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233ae2400. 00010000:00000010:2.1:1775773750.787180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bef000. 00010000:00000010:2.1:1775773750.787182:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62052b3700. 00010000:00000010:2.1:1775773750.787184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233befe00. 00010000:00000010:2.1:1775773750.787186:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62052b3a00. 00010000:00000010:2.1:1775773750.787187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e62ac00. 00010000:00000010:2.1:1775773750.787189:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621ddafe00. 00010000:00000010:2.1:1775773750.787191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62067b2400. 00010000:00000010:2.1:1775773750.787192:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621ddaf900. 00010000:00000010:2.1:1775773750.787194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62067b3600. 00010000:00000010:2.1:1775773750.787195:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62052b3100. 00010000:00000010:2.1:1775773750.787197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bef200. 00010000:00000010:2.1:1775773750.787199:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62052b3600. 00010000:00000010:2.1:1775773750.787201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bee400. 00010000:00000010:2.1:1775773750.787203:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233ae2800. 00010000:00000010:2.1:1775773750.787205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62066da800. 00010000:00000010:2.1:1775773750.787207:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233ae2e00. 00010000:00000010:2.1:1775773750.787212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62066db800. 00010000:00000010:2.1:1775773750.787215:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349b8400. 00010000:00000010:2.1:1775773750.787217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bee800. 00010000:00000010:2.1:1775773750.787219:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62052b3f00. 00010000:00000010:2.1:1775773750.787221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bef800. 00010000:00000010:2.1:1775773750.787223:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62052b3000. 00010000:00000010:2.1:1775773750.787225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bee600. 00010000:00000010:2.1:1775773750.787227:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62052b3500. 00010000:00000010:2.1:1775773750.787229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206227a00. 00010000:00000010:2.1:1775773750.787232:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203225200. 00010000:00000010:2.1:1775773750.787234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206227000. 00010000:00000010:2.1:1775773750.787235:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203225d00. 00010000:00000010:2.1:1775773750.787237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206227e00. 00010000:00000010:2.1:1775773750.787239:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349b8800. 00010000:00000010:2.1:1775773750.787241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205728e00. 00010000:00000010:2.1:1775773750.787243:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621ddaf700. 00010000:00000010:2.1:1775773750.787244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e62a200. 00010000:00000010:2.1:1775773750.787246:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349b8e00. 00010000:00000010:2.1:1775773750.787248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e62be00. 00010000:00000010:2.1:1775773750.787250:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203225400. 00010000:00000010:2.1:1775773750.787251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206227200. 00010000:00000010:2.1:1775773750.787253:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621ddafa00. 00010000:00000010:2.1:1775773750.787255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e62a400. 00010000:00000010:2.1:1775773750.787256:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349b8900. 00010000:00000010:2.1:1775773750.787258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205728200. 00010000:00000010:2.1:1775773750.787261:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203225800. 00010000:00000010:2.1:1775773750.787263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206226400. 00010000:00000010:2.1:1775773750.787264:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203225e00. 00010000:00000010:2.1:1775773750.787267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e62b000. 00010000:00000010:2.1:1775773750.787268:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349b8700. 00010000:00000010:2.1:1775773750.787270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62067b2800. 00010000:00000010:2.1:1775773750.787272:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203225900. 00010000:00000010:2.1:1775773750.787274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206226800. 00010000:00000010:2.1:1775773750.787276:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349b8a00. 00010000:00000010:2.1:1775773750.787278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205729a00. 00010000:00000010:2.1:1775773750.787280:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349b8100. 00010000:00000010:2.1:1775773750.787282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205729000. 00010000:00000010:2.1:1775773750.787285:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203225700. 00010000:00000010:2.1:1775773750.787288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206227800. 00010000:00000010:2.1:1775773750.787290:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349b8600. 00010000:00000010:2.1:1775773750.787293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206226600. 00010000:00000010:2.1:1775773750.787295:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203225a00. 00010000:00000010:2.1:1775773750.787298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206226000. 00010000:00000010:2.1:1775773750.787301:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203225100. 00010000:00000010:2.1:1775773750.787303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206226a00. 00010000:00000010:2.1:1775773750.787306:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203225600. 00010000:00000010:2.1:1775773750.787309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206227400. 00010000:00000010:2.1:1775773750.787312:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203225f00. 00010000:00000010:2.1:1775773750.787315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206227c00. 00010000:00000010:2.1:1775773750.787318:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203225000. 00010000:00000010:2.1:1775773750.787321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206226c00. 00010000:00000010:2.1:1775773750.787323:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203225500. 00010000:00000010:2.1:1775773750.787326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206227600. 00010000:00000010:2.1:1775773750.787329:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203225b00. 00010000:00000010:2.1:1775773750.787332:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206226e00. 00010000:00000010:2.1:1775773750.787335:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349b8f00. 00010000:00000010:2.1:1775773750.787338:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e62ba00. 00010000:00000010:2.1:1775773750.787340:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203225c00. 00010000:00000010:2.1:1775773750.787342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206226200. 00010000:00000010:2.1:1775773750.787345:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6203225300. 00010000:00000010:2.1:1775773750.787348:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e62b200. 00010000:00000010:2.1:1775773750.787351:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bc9ec00. 00010000:00000010:2.1:1775773750.787354:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e62a600. 00010000:00000010:2.1:1775773750.787356:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349b8000. 00010000:00000010:2.1:1775773750.787359:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205729e00. 00010000:00000010:2.1:1775773750.787363:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae0100. 00010000:00000010:2.1:1775773750.787366:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e62a000. 00010000:00000010:2.1:1775773750.787369:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349b8500. 00010000:00000010:2.1:1775773750.787371:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205729200. 00010000:00000010:2.1:1775773750.787374:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621ddaf600. 00010000:00000010:2.1:1775773750.787385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62067b3e00. 00010000:00000010:2.1:1775773750.787389:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae0600. 00010000:00000010:2.1:1775773750.787392:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddc5200. 00010000:00000010:2.1:1775773750.787396:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e106700. 00010000:00000010:2.1:1775773750.787398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62067b3200. 00010000:00000010:2.1:1775773750.787401:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e106a00. 00010000:00000010:2.1:1775773750.787416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62067b2600. 00010000:00000010:2.1:1775773750.787420:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae0f00. 00010000:00000010:2.1:1775773750.787423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e62a800. 00010000:00000010:2.1:1775773750.787431:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae0000. 00010000:00000010:2.1:1775773750.787435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddc4400. 00010000:00000010:2.1:1775773750.787438:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae0500. 00010000:00000010:2.1:1775773750.787440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddc4800. 00010000:00000010:2.1:1775773750.787443:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae0b00. 00010000:00000010:2.1:1775773750.787446:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddc5800. 00010000:00000010:2.1:1775773750.787449:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae0c00. 00010000:00000010:2.1:1775773750.787452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddc4600. 00010000:00000010:2.1:1775773750.787454:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae0300. 00010000:00000010:2.1:1775773750.787457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddc4000. 00010000:00000010:2.1:1775773750.787459:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae0200. 00010000:00000010:2.1:1775773750.787462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddc4a00. 00010000:00000010:2.1:1775773750.787465:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349b8b00. 00010000:00000010:2.1:1775773750.787468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205728400. 00010000:00000010:2.1:1775773750.787471:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349b8c00. 00010000:00000010:2.1:1775773750.787474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205728800. 00010000:00000010:2.1:1775773750.787477:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae0d00. 00010000:00000010:2.1:1775773750.787480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddc5400. 00010000:00000010:2.1:1775773750.787482:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae0400. 00010000:00000010:2.1:1775773750.787485:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62067b2a00. 00010000:00000010:2.1:1775773750.787488:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e106100. 00010000:00000010:2.1:1775773750.787491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62108b9600. 00010000:00000010:2.1:1775773750.787494:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae0800. 00010000:00000010:2.1:1775773750.787497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddc5c00. 00010000:00000010:2.1:1775773750.787500:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e106600. 00010000:00000010:2.1:1775773750.787503:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62067b2c00. 00010000:00000010:2.1:1775773750.787505:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e106f00. 00010000:00000010:2.1:1775773750.787508:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddc4c00. 00010000:00000010:2.1:1775773750.787510:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae0e00. 00010000:00000010:2.1:1775773750.787513:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddc5600. 00010000:00000010:2.1:1775773750.787515:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349b8300. 00010000:00000010:2.1:1775773750.787518:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205729800. 00010000:00000010:2.1:1775773750.787521:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349b8200. 00010000:00000010:2.1:1775773750.787523:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205728000. 00010000:00000010:2.1:1775773750.787526:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae0900. 00010000:00000010:2.1:1775773750.787529:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddc4e00. 00010000:00000010:2.1:1775773750.787531:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e106000. 00010000:00000010:2.1:1775773750.787534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621079e800. 00010000:00000010:2.1:1775773750.787536:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae0700. 00010000:00000010:2.1:1775773750.787538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddc4200. 00010000:00000010:2.1:1775773750.787540:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225ae0a00. 00010000:00000010:2.1:1775773750.787541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddc5a00. 00010000:00000010:2.1:1775773750.787544:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202e77c00. 00010000:00000010:2.1:1775773750.787546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddc5000. 00010000:00000010:2.1:1775773750.787547:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202e77300. 00010000:00000010:2.1:1775773750.787550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62108b9200. 00010000:00000010:2.1:1775773750.787551:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e106500. 00010000:00000010:2.1:1775773750.787553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621079e600. 00010000:00000010:2.1:1775773750.787555:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202e77200. 00010000:00000010:2.1:1775773750.787557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddc5e00. 00010000:00000010:2.1:1775773750.787559:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202e77d00. 00010000:00000010:2.1:1775773750.787560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dc09600. 00010000:00000010:2.1:1775773750.787562:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202e77400. 00010000:00000010:2.1:1775773750.787564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dc08e00. 00010000:00000010:2.1:1775773750.787565:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e106b00. 00010000:00000010:2.1:1775773750.787567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621079fc00. 00010000:00000010:2.1:1775773750.787568:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e106c00. 00010000:00000010:2.1:1775773750.787570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621079f000. 00010000:00000010:2.1:1775773750.787571:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e106300. 00010000:00000010:2.1:1775773750.787573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621079f600. 00010000:00000010:2.1:1775773750.787575:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202e77800. 00010000:00000010:2.1:1775773750.787576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dc08200. 00010000:00000010:2.1:1775773750.787578:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202e77e00. 00010000:00000010:2.1:1775773750.787579:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621079e400. 00010000:00000010:2.1:1775773750.787581:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202e77900. 00010000:00000010:2.1:1775773750.787583:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dc09a00. 00010000:00000010:2.1:1775773750.787584:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e106200. 00010000:00000010:2.1:1775773750.787586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621079f200. 00010000:00000010:2.1:1775773750.787587:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349b8d00. 00010000:00000010:2.1:1775773750.787589:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6205728600. 00010000:00000010:2.1:1775773750.787595:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202e77700. 00010000:00000010:2.1:1775773750.787597:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dc09000. 00010000:00000010:2.1:1775773750.787599:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e106d00. 00010000:00000010:2.1:1775773750.787603:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621079e000. 00010000:00000010:2.1:1775773750.787607:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621193a600. 00010000:00000010:2.1:1775773750.787611:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62108b9400. 00010000:00000010:2.1:1775773750.787616:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202e77a00. 00010000:00000010:2.1:1775773750.787620:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dc09e00. 00000080:00000010:2.1:1775773761.011182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04600. 00000080:00000010:2.1:1775773761.011197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04ee0. 00000080:00000010:2.1:1775773761.015177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa680. 00000080:00000010:2.1:1775773761.015190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fda00. 00000080:00000010:2.1:1775773761.015197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04ac0. 00000080:00000010:2.1:1775773761.015201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b7c0. 00000080:00000010:2.1:1775773761.015206:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fa40. 00000080:00000010:2.1:1775773761.015210:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04a40. 00000080:00000010:2.1:1775773761.015213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdbe0. 00000080:00000010:2.1:1775773761.015216:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04a00. 00000080:00000010:2.1:1775773761.015219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04440. 00000080:00000010:2.1:1775773761.015222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f320. 00000080:00000010:2.1:1775773761.015245:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04c60. 00000080:00000010:2.1:1775773761.015249:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd140. 00000080:00000010:2.1:1775773761.015252:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04560. 00000080:00000010:2.1:1775773761.015255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa3c0. 00000080:00000010:2.1:1775773761.015257:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa080. 00000080:00000010:2.1:1775773761.015260:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b040c0. 00000080:00000010:2.1:1775773761.015263:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04f20. 00000080:00000010:2.1:1775773761.015268:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04f00. 00000080:00000010:2.1:1775773761.015271:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04fa0. 00000080:00000010:2.1:1775773761.015274:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04460. 00000080:00000010:2.1:1775773761.015277:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04dc0. 00000080:00000010:2.1:1775773761.015280:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04a20. 00000080:00000010:2.1:1775773761.015283:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b046c0. 00000080:00000010:2.1:1775773761.015290:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4ae0. 00000080:00000010:2.1:1775773761.015293:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f020. 00000080:00000010:2.1:1775773761.015295:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fc20. 00000080:00000010:2.1:1775773761.015298:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b042e0. 00000080:00000010:2.1:1775773761.015304:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4dc0. 00000080:00000010:2.1:1775773761.015307:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04aa0. 00000080:00000010:2.1:1775773761.015310:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04a60. 00000080:00000010:2.1:1775773761.015312:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4c00. 00000080:00000010:2.1:1775773761.015315:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04160. 00000080:00000010:2.1:1775773761.015319:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04d60. 00000080:00000010:2.1:1775773761.015322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b046e0. 00000080:00000010:2.1:1775773761.015324:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04140. 00000080:00000010:2.1:1775773761.015327:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04060. 00000080:00000010:2.1:1775773761.015330:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04e80. 00000080:00000010:2.1:1775773761.015333:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04f40. 00000080:00000010:2.1:1775773761.015335:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04800. 00000080:00000010:2.1:1775773761.015338:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db49a0. 00000080:00000010:2.1:1775773761.015341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04ce0. 00000080:00000010:2.1:1775773761.015343:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b042c0. 00000080:00000010:2.1:1775773761.015346:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04340. 00000080:00000010:2.1:1775773761.015348:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04000. 00000080:00000010:2.1:1775773761.015351:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04420. 00000080:00000010:2.1:1775773761.015353:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4280. 00000080:00000010:2.1:1775773761.015356:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057ebec0. 00000080:00000010:2.1:1775773761.015359:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd12460. 00000080:00000010:2.1:1775773761.015362:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04680. 00000080:00000010:2.1:1775773761.015367:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b043a0. 00000080:00000010:2.1:1775773761.015370:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04400. 00000080:00000010:2.1:1775773761.015373:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4e20. 00000080:00000010:2.1:1775773761.015376:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b045c0. 00000080:00000010:2.1:1775773761.015378:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233edbba0. 00000080:00000010:2.1:1775773761.015381:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd12500. 00000080:00000010:2.1:1775773761.015385:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04280. 00000080:00000010:2.1:1775773761.015387:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd12e20. 00000080:00000010:2.1:1775773761.015390:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fb00. 00000080:00000010:2.1:1775773761.015393:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd12e80. 00000080:00000010:2.1:1775773761.015395:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04180. 00000080:00000010:2.1:1775773761.015398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04e60. 00000080:00000010:2.1:1775773761.015401:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04fc0. 00000080:00000010:2.1:1775773761.015403:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd12040. 00000080:00000010:2.1:1775773761.015406:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd12a40. 00000080:00000010:2.1:1775773761.015408:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04200. 00000080:00000010:2.1:1775773761.015411:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04c20. 00000080:00000010:2.1:1775773761.015414:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04ba0. 00000080:00000010:2.1:1775773761.015416:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b047a0. 00000080:00000010:2.1:1775773761.015419:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04a80. 00000080:00000010:2.1:1775773761.015422:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b048a0. 00000080:00000010:2.1:1775773761.015425:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b044a0. 00000080:00000010:2.1:1775773761.015427:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f700. 00000080:00000010:2.1:1775773761.015430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b048e0. 00000080:00000010:2.1:1775773761.015432:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b041a0. 00000080:00000010:2.1:1775773761.015437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04220. 00000080:00000010:2.1:1775773761.015439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04920. 00000080:00000010:2.1:1775773761.015442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04bc0. 00000080:00000010:2.1:1775773761.015444:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b044c0. 00000080:00000010:2.1:1775773761.015447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b043c0. 00000080:00000010:2.1:1775773761.015449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4d00. 00000080:00000010:2.1:1775773761.015453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f840. 00000080:00000010:2.1:1775773761.015456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4160. 00000080:00000010:2.1:1775773761.015459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4520. 00000080:00000010:2.1:1775773761.015461:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4de0. 00000080:00000010:2.1:1775773761.015464:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04260. 00000080:00000010:2.1:1775773761.015466:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db49c0. 00000080:00000010:2.1:1775773761.015469:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fb20. 00000080:00000010:2.1:1775773761.015472:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fae0. 00000080:00000010:2.1:1775773761.015474:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f460. 00000080:00000010:2.1:1775773761.015477:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4bc0. 00000080:00000010:2.1:1775773761.015479:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd12fe0. 00000080:00000010:2.1:1775773761.015482:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f160. 00000080:00000010:2.1:1775773761.015484:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fc60. 00000080:00000010:2.1:1775773761.015487:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db42c0. 00000080:00000010:2.1:1775773761.015490:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b045a0. 00000080:00000010:2.1:1775773761.015493:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f540. 00000080:00000010:2.1:1775773761.015495:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fda0. 00000080:00000010:2.1:1775773761.015498:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db47c0. 00000080:00000010:2.1:1775773761.015500:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04080. 00000080:00000010:2.1:1775773761.015502:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd12e40. 00000080:00000010:2.1:1775773761.015505:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04640. 00000080:00000010:2.1:1775773761.015507:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd12320. 00000080:00000010:2.1:1775773761.015510:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f580. 00000080:00000010:2.1:1775773761.015512:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04960. 00000080:00000010:2.1:1775773761.015515:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4320. 00000080:00000010:2.1:1775773761.015518:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04ca0. 00000080:00000010:2.1:1775773761.015520:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04120. 00000080:00000010:2.1:1775773761.015522:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b040e0. 00000080:00000010:2.1:1775773761.015525:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04300. 00000080:00000010:2.1:1775773761.015527:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd12d80. 00000080:00000010:2.1:1775773761.015529:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04c80. 00000080:00000010:2.1:1775773761.015532:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04820. 00000080:00000010:2.1:1775773761.015535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b046a0. 00000080:00000010:2.1:1775773761.015537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd12c40. 00000080:00000010:2.1:1775773761.015539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04ec0. 00000080:00000010:2.1:1775773761.015541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04de0. 00000080:00000010:2.1:1775773761.015544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b049e0. 00000080:00000010:2.1:1775773761.015546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd127e0. 00000080:00000010:2.1:1775773761.015548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04360. 00000080:00000010:2.1:1775773761.015550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b044e0. 00000080:00000010:2.1:1775773761.015552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04780. 00000080:00000010:2.1:1775773761.015555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f880. 00000080:00000010:2.1:1775773761.015557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04d40. 00000080:00000010:2.1:1775773761.015559:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04740. 00000080:00000010:2.1:1775773761.015561:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4a40. 00000080:00000010:2.1:1775773761.015563:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b047c0. 00000080:00000010:2.1:1775773761.015566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f820. 00000080:00000010:2.1:1775773761.015568:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04cc0. 00000080:00000010:2.1:1775773761.015574:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b049a0. 00000080:00000010:2.1:1775773761.015577:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db42e0. 00000080:00000010:2.1:1775773761.015579:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4120. 00000080:00000010:2.1:1775773761.015581:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04520. 00000080:00000010:2.1:1775773761.015584:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04b00. 00000080:00000010:2.1:1775773761.015586:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b043e0. 00000080:00000010:2.1:1775773761.015588:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fdc0. 00000080:00000010:2.1:1775773761.015590:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4660. 00000080:00000010:2.1:1775773761.015592:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04860. 00000080:00000010:2.1:1775773761.015595:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4ac0. 00000080:00000010:2.1:1775773761.015597:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04d80. 00000080:00000010:2.1:1775773761.015599:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f080. 00000080:00000010:2.1:1775773761.015601:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4680. 00000080:00000010:2.1:1775773761.015604:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04fe0. 00000080:00000010:2.1:1775773761.015606:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f680. 00000080:00000010:2.1:1775773761.015608:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4640. 00000080:00000010:2.1:1775773761.015611:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04100. 00000080:00000010:2.1:1775773761.015613:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c6cc2a0. 00000080:00000010:2.1:1775773761.015615:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fa20. 00000080:00000010:2.1:1775773761.015618:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd121e0. 00000080:00000010:2.1:1775773761.015620:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792c4e0. 00000080:00000010:2.1:1775773761.015623:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792c440. 00000080:00000010:2.1:1775773761.015625:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f6c0. 00000080:00000010:2.1:1775773761.015627:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792ccc0. 00000080:00000010:2.1:1775773761.015629:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792caa0. 00000080:00000010:2.1:1775773761.015631:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62064a2f40. 00000080:00000010:2.1:1775773761.015633:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f800. 00000080:00000010:2.1:1775773761.015635:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f640. 00000080:00000010:2.1:1775773761.015638:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f980. 00000080:00000010:2.1:1775773761.015640:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4720. 00000080:00000010:2.1:1775773761.015642:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f220. 00000080:00000010:2.1:1775773761.015644:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0ff60. 00000080:00000010:2.1:1775773761.015646:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f0e0. 00000080:00000010:2.1:1775773761.015648:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd12fa0. 00000080:00000010:2.1:1775773761.015650:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4a00. 00000080:00000010:2.1:1775773761.015652:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f720. 00000080:00000010:2.1:1775773761.015654:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f960. 00000080:00000010:2.1:1775773761.015656:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93540. 00000080:00000010:2.1:1775773761.015659:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233edb880. 00000080:00000010:2.1:1775773761.015661:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620a9bf3e0. 00000080:00000010:2.1:1775773761.015663:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93a40. 00000080:00000010:2.1:1775773761.015665:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4420. 00000080:00000010:2.1:1775773761.015667:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93860. 00000080:00000010:2.1:1775773761.015669:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93420. 00000080:00000010:2.1:1775773761.015671:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93b20. 00000080:00000010:2.1:1775773761.015673:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b1e0. 00010000:00000010:2.1:1775773761.015677:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225922c00. 00010000:00000010:2.1:1775773761.015681:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234917c00. 00010000:00000010:2.1:1775773761.015700:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225922300. 00010000:00000010:2.1:1775773761.015702:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234916c00. 00010000:00000010:2.1:1775773761.015704:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623386cc00. 00010000:00000010:2.1:1775773761.015706:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bfbda00. 00010000:00000010:2.1:1775773761.015709:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226601300. 00010000:00000010:2.1:1775773761.015711:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa5200. 00010000:00000010:2.1:1775773761.015713:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225922200. 00010000:00000010:2.1:1775773761.015715:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234917600. 00010000:00000010:2.1:1775773761.015717:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab16200. 00010000:00000010:2.1:1775773761.015719:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bfbd000. 00010000:00000010:2.1:1775773761.015720:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225922d00. 00010000:00000010:2.1:1775773761.015722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226684800. 00010000:00000010:2.1:1775773761.015724:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225922400. 00010000:00000010:2.1:1775773761.015726:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234916e00. 00010000:00000010:2.1:1775773761.015728:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226601200. 00010000:00000010:2.1:1775773761.015730:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa4000. 00010000:00000010:2.1:1775773761.015732:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfc8300. 00010000:00000010:2.1:1775773761.015734:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234916200. 00010000:00000010:2.1:1775773761.015735:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225922800. 00010000:00000010:2.1:1775773761.015737:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234917a00. 00010000:00000010:2.1:1775773761.015739:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfc8000. 00010000:00000010:2.1:1775773761.015741:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226684c00. 00010000:00000010:2.1:1775773761.015742:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225922e00. 00010000:00000010:2.1:1775773761.015744:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234917000. 00010000:00000010:2.1:1775773761.015746:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226601d00. 00010000:00000010:2.1:1775773761.015748:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa4200. 00010000:00000010:2.1:1775773761.015750:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225922900. 00010000:00000010:2.1:1775773761.015751:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234917e00. 00010000:00000010:2.1:1775773761.015753:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225922700. 00010000:00000010:2.1:1775773761.015755:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bfbde00. 00010000:00000010:2.1:1775773761.015757:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623386c300. 00010000:00000010:2.1:1775773761.015759:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bfbd200. 00010000:00000010:2.1:1775773761.015761:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226601400. 00010000:00000010:2.1:1775773761.015763:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234917200. 00010000:00000010:2.1:1775773761.015765:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225922a00. 00010000:00000010:2.1:1775773761.015767:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234916400. 00010000:00000010:2.1:1775773761.015768:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225922100. 00010000:00000010:2.1:1775773761.015770:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234916800. 00010000:00000010:2.1:1775773761.015772:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd95400. 00010000:00000010:2.1:1775773761.015774:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234917800. 00010000:00000010:2.1:1775773761.015776:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd95800. 00010000:00000010:2.1:1775773761.015777:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234916600. 00010000:00000010:2.1:1775773761.015779:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd95e00. 00010000:00000010:2.1:1775773761.015781:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234916000. 00010000:00000010:2.1:1775773761.015782:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd95900. 00010000:00000010:2.1:1775773761.015784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234916a00. 00010000:00000010:2.1:1775773761.015786:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd95700. 00010000:00000010:2.1:1775773761.015787:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234917400. 00010000:00000010:2.1:1775773761.015789:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd95a00. 00010000:00000010:2.1:1775773761.015791:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa5c00. 00010000:00000010:2.1:1775773761.015792:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfc8500. 00010000:00000010:2.1:1775773761.015794:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226685000. 00010000:00000010:2.1:1775773761.015796:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfc8b00. 00010000:00000010:2.1:1775773761.015798:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226685600. 00010000:00000010:2.1:1775773761.015800:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd95100. 00010000:00000010:2.1:1775773761.015801:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672fe00. 00010000:00000010:2.1:1775773761.015803:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226601800. 00010000:00000010:2.1:1775773761.015805:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa4400. 00010000:00000010:2.1:1775773761.015807:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd95600. 00010000:00000010:2.1:1775773761.015808:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672f200. 00010000:00000010:2.1:1775773761.015810:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd95f00. 00010000:00000010:2.1:1775773761.015812:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672e400. 00010000:00000010:2.1:1775773761.015814:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226601e00. 00010000:00000010:2.1:1775773761.015815:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa5000. 00010000:00000010:2.1:1775773761.015818:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226601900. 00010000:00000010:2.1:1775773761.015819:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672e800. 00010000:00000010:2.1:1775773761.015821:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd95000. 00010000:00000010:2.1:1775773761.015823:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672f800. 00010000:00000010:2.1:1775773761.015825:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623386c200. 00010000:00000010:2.1:1775773761.015827:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672e600. 00010000:00000010:2.1:1775773761.015828:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd95500. 00010000:00000010:2.1:1775773761.015830:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672e000. 00010000:00000010:2.1:1775773761.015831:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd95b00. 00010000:00000010:2.1:1775773761.015833:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672ea00. 00010000:00000010:2.1:1775773761.015835:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd95c00. 00010000:00000010:2.1:1775773761.015836:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672f400. 00010000:00000010:2.1:1775773761.015838:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd95300. 00010000:00000010:2.1:1775773761.015839:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672fc00. 00010000:00000010:2.1:1775773761.015841:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd95200. 00010000:00000010:2.1:1775773761.015842:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672ec00. 00010000:00000010:2.1:1775773761.015844:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226601700. 00010000:00000010:2.1:1775773761.015846:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa4e00. 00010000:00000010:2.1:1775773761.015848:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226601a00. 00010000:00000010:2.1:1775773761.015849:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa5a00. 00010000:00000010:2.1:1775773761.015851:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd95d00. 00010000:00000010:2.1:1775773761.015852:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672f600. 00010000:00000010:2.1:1775773761.015854:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205083300. 00010000:00000010:2.1:1775773761.015856:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672ee00. 00010000:00000010:2.1:1775773761.015858:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e339e00. 00010000:00000010:2.1:1775773761.015860:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672e200. 00010000:00000010:2.1:1775773761.015862:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e339900. 00010000:00000010:2.1:1775773761.015864:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672fa00. 00010000:00000010:2.1:1775773761.015865:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226601100. 00010000:00000010:2.1:1775773761.015867:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa4c00. 00010000:00000010:2.1:1775773761.015869:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205083200. 00010000:00000010:2.1:1775773761.015870:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622672f000. 00010000:00000010:2.1:1775773761.015872:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e339700. 00010000:00000010:2.1:1775773761.015875:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bfbc400. 00010000:00000010:2.1:1775773761.015876:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e339a00. 00010000:00000010:2.1:1775773761.015878:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f624206c200. 00010000:00000010:2.1:1775773761.015881:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205083d00. 00010000:00000010:2.1:1775773761.015883:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f624206d400. 00010000:00000010:2.1:1775773761.015884:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205083400. 00010000:00000010:2.1:1775773761.015886:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f624206dc00. 00010000:00000010:2.1:1775773761.015888:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205083800. 00010000:00000010:2.1:1775773761.015889:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa5400. 00010000:00000010:2.1:1775773761.015891:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205083e00. 00010000:00000010:2.1:1775773761.015893:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f624206cc00. 00010000:00000010:2.1:1775773761.015894:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226601600. 00010000:00000010:2.1:1775773761.015896:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bfbc800. 00010000:00000010:2.1:1775773761.015897:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e339100. 00010000:00000010:2.1:1775773761.015899:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bfbd800. 00010000:00000010:2.1:1775773761.015901:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e339600. 00010000:00000010:2.1:1775773761.015902:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f624206d600. 00010000:00000010:2.1:1775773761.015904:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e339f00. 00010000:00000010:2.1:1775773761.015906:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bfbc600. 00010000:00000010:2.1:1775773761.015907:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfc8c00. 00010000:00000010:2.1:1775773761.015909:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226685c00. 00010000:00000010:2.1:1775773761.015911:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e339000. 00010000:00000010:2.1:1775773761.015912:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bfbc000. 00010000:00000010:2.1:1775773761.015914:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205083900. 00010000:00000010:2.1:1775773761.015916:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f624206ce00. 00010000:00000010:2.1:1775773761.015917:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205083700. 00010000:00000010:2.1:1775773761.015919:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f624206da00. 00010000:00000010:2.1:1775773761.015920:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205083a00. 00010000:00000010:2.1:1775773761.015922:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f624206d000. 00010000:00000010:2.1:1775773761.015924:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e339500. 00010000:00000010:2.1:1775773761.015925:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bfbca00. 00010000:00000010:2.1:1775773761.015927:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e339b00. 00010000:00000010:2.1:1775773761.015929:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62066da200. 00010000:00000010:2.1:1775773761.015932:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205083100. 00010000:00000010:2.1:1775773761.015934:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f624206de00. 00010000:00000010:2.1:1775773761.015935:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205083600. 00010000:00000010:2.1:1775773761.015937:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f624206d200. 00010000:00000010:2.1:1775773761.015939:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205083f00. 00010000:00000010:2.1:1775773761.015940:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f624206c400. 00010000:00000010:2.1:1775773761.015942:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205083000. 00010000:00000010:2.1:1775773761.015943:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f624206c800. 00010000:00000010:2.1:1775773761.015945:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205083500. 00010000:00000010:2.1:1775773761.015947:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f624206d800. 00010000:00000010:2.1:1775773761.015948:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205083b00. 00010000:00000010:2.1:1775773761.015950:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f624206c600. 00010000:00000010:2.1:1775773761.015951:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205083c00. 00010000:00000010:2.1:1775773761.015953:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f624206c000. 00010000:00000010:2.1:1775773761.015954:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226601f00. 00010000:00000010:2.1:1775773761.015956:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226684a00. 00010000:00000010:2.1:1775773761.015958:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225b39700. 00010000:00000010:2.1:1775773761.015960:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f624206ca00. 00010000:00000010:2.1:1775773761.015962:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225b39a00. 00010000:00000010:2.1:1775773761.015964:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211ba6a00. 00010000:00000010:2.1:1775773761.015966:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e339c00. 00010000:00000010:2.1:1775773761.015967:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211ba7400. 00010000:00000010:2.1:1775773761.015969:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225b39100. 00010000:00000010:2.1:1775773761.015971:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211ba7c00. 00010000:00000010:2.1:1775773761.015973:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225b39600. 00010000:00000010:2.1:1775773761.015974:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211ba6c00. 00010000:00000010:2.1:1775773761.015976:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225b39f00. 00010000:00000010:2.1:1775773761.015978:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211ba7600. 00010000:00000010:2.1:1775773761.015979:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225b39000. 00010000:00000010:2.1:1775773761.015981:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211ba6e00. 00010000:00000010:2.1:1775773761.015983:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225b39500. 00010000:00000010:2.1:1775773761.015985:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa5600. 00010000:00000010:2.1:1775773761.015987:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e339300. 00010000:00000010:2.1:1775773761.015988:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226685800. 00010000:00000010:2.1:1775773761.015990:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226601000. 00010000:00000010:2.1:1775773761.015992:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa4a00. 00010000:00000010:2.1:1775773761.015994:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e339200. 00010000:00000010:2.1:1775773761.015995:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa5800. 00010000:00000010:2.1:1775773761.015997:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225b39b00. 00010000:00000010:2.1:1775773761.015999:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa4800. 00010000:00000010:2.1:1775773761.016000:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225b39c00. 00010000:00000010:2.1:1775773761.016002:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211ba6200. 00010000:00000010:2.1:1775773761.016003:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226601500. 00010000:00000010:2.1:1775773761.016005:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa5e00. 00010000:00000010:2.1:1775773761.016007:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e339d00. 00010000:00000010:2.1:1775773761.016008:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226684e00. 00010000:00000010:2.1:1775773761.016010:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225b39300. 00010000:00000010:2.1:1775773761.016011:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226684200. 00010000:00000010:2.1:1775773761.016013:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e339400. 00010000:00000010:2.1:1775773761.016014:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226685a00. 00010000:00000010:2.1:1775773761.016016:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226601b00. 00010000:00000010:2.1:1775773761.016018:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c707800. 00010000:00000010:2.1:1775773761.016020:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e339800. 00010000:00000010:2.1:1775773761.016021:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62066da600. 00010000:00000010:2.1:1775773761.016023:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225b39200. 00010000:00000010:2.1:1775773761.016025:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226685400. 00010000:00000010:2.1:1775773761.016026:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfc8200. 00010000:00000010:2.1:1775773761.016028:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623395fa00. 00010000:00000010:2.1:1775773761.016046:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620adff200. 00010000:00000010:2.1:1775773761.016048:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c706a00. 00010000:00000010:2.1:1775773761.016050:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225b39d00. 00010000:00000010:2.1:1775773761.016051:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211ba7a00. 00010000:00000010:2.1:1775773761.016053:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfc8d00. 00010000:00000010:2.1:1775773761.016055:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623395f000. 00010000:00000010:2.1:1775773761.016057:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225b39400. 00010000:00000010:2.1:1775773761.016059:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623395f400. 00010000:00000010:2.1:1775773761.016061:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205044900. 00010000:00000010:2.1:1775773761.016063:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c707e00. 00010000:00000010:2.1:1775773761.016064:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225b39800. 00010000:00000010:2.1:1775773761.016066:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211ba7000. 00010000:00000010:2.1:1775773761.016067:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205044700. 00010000:00000010:2.1:1775773761.016069:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62066da000. 00010000:00000010:2.1:1775773761.016070:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225b39e00. 00010000:00000010:2.1:1775773761.016072:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211ba7e00. 00010000:00000010:2.1:1775773761.016073:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620adffd00. 00010000:00000010:2.1:1775773761.016075:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62066daa00. 00010000:00000010:2.1:1775773761.016077:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225b39900. 00010000:00000010:2.1:1775773761.016078:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623395e000. 00010000:00000010:2.1:1775773761.016080:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa9400. 00010000:00000010:2.1:1775773761.016082:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211ba7200. 00010000:00000010:2.1:1775773761.016084:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205044a00. 00010000:00000010:2.1:1775773761.016085:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c706e00. 00010000:00000010:2.1:1775773761.016087:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa9800. 00010000:00000010:2.1:1775773761.016088:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211ba6400. 00010000:00000010:2.1:1775773761.016090:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa9e00. 00010000:00000010:2.1:1775773761.016092:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211ba6800. 00010000:00000010:2.1:1775773761.016093:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa9900. 00010000:00000010:2.1:1775773761.016095:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211ba7800. 00010000:00000010:2.1:1775773761.016096:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa9700. 00010000:00000010:2.1:1775773761.016098:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211ba6600. 00010000:00000010:2.1:1775773761.016100:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa9a00. 00010000:00000010:2.1:1775773761.016101:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62066db400. 00010000:00000010:2.1:1775773761.016103:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa9100. 00010000:00000010:2.1:1775773761.016105:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211ba6000. 00010000:00000010:2.1:1775773761.016107:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa9600. 00010000:00000010:2.1:1775773761.016109:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e58a200. 00010000:00000010:2.1:1775773761.016111:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa9f00. 00010000:00000010:2.1:1775773761.016113:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e58ba00. 00010000:00000010:2.1:1775773761.016115:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620adff400. 00010000:00000010:2.1:1775773761.016117:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62066dbc00. 00010000:00000010:2.1:1775773761.016118:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa9000. 00010000:00000010:2.1:1775773761.016120:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e58b000. 00010000:00000010:2.1:1775773761.016121:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa9500. 00010000:00000010:2.1:1775773761.016123:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e58be00. 00010000:00000010:2.1:1775773761.016124:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa9b00. 00010000:00000010:2.1:1775773761.016126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e58b200. 00010000:00000010:2.1:1775773761.016128:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620adff800. 00010000:00000010:2.1:1775773761.016129:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62066dac00. 00010000:00000010:2.1:1775773761.016131:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa9c00. 00010000:00000010:2.1:1775773761.016132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e58a400. 00010000:00000010:2.1:1775773761.016134:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfc8400. 00010000:00000010:2.1:1775773761.016135:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e58a800. 00010000:00000010:2.1:1775773761.016137:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa9300. 00010000:00000010:2.1:1775773761.016139:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e58b800. 00010000:00000010:2.1:1775773761.016140:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfc8800. 00010000:00000010:2.1:1775773761.016142:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623395ec00. 00010000:00000010:2.1:1775773761.016143:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa9200. 00010000:00000010:2.1:1775773761.016145:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e58a600. 00010000:00000010:2.1:1775773761.016146:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa9d00. 00010000:00000010:2.1:1775773761.016148:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e58a000. 00010000:00000010:2.1:1775773761.016149:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205044100. 00010000:00000010:2.1:1775773761.016151:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c706800. 00010000:00000010:2.1:1775773761.016153:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349aeb00. 00010000:00000010:2.1:1775773761.016203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e58aa00. 00010000:00000010:2.1:1775773761.016206:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfc8e00. 00010000:00000010:2.1:1775773761.016208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623395e800. 00010000:00000010:2.1:1775773761.016211:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620adffe00. 00010000:00000010:2.1:1775773761.016212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e58b400. 00010000:00000010:2.1:1775773761.016215:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349aec00. 00010000:00000010:2.1:1775773761.016216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e58bc00. 00010000:00000010:2.1:1775773761.016218:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfc8900. 00010000:00000010:2.1:1775773761.016220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c707a00. 00010000:00000010:2.1:1775773761.016222:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620adff900. 00010000:00000010:2.1:1775773761.016223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c706400. 00010000:00000010:2.1:1775773761.016244:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349ae300. 00010000:00000010:2.1:1775773761.016246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e58ac00. 00010000:00000010:2.1:1775773761.016248:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349ae200. 00010000:00000010:2.1:1775773761.016249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e58b600. 00010000:00000010:2.1:1775773761.016251:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349aed00. 00010000:00000010:2.1:1775773761.016252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e58ae00. 00010000:00000010:2.1:1775773761.016254:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349ae400. 00010000:00000010:2.1:1775773761.016256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623395e400. 00010000:00000010:2.1:1775773761.016257:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfc8700. 00010000:00000010:2.1:1775773761.016259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c706c00. 00010000:00000010:2.1:1775773761.016260:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349ae800. 00010000:00000010:2.1:1775773761.016262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623441a800. 00010000:00000010:2.1:1775773761.016264:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620adff700. 00010000:00000010:2.1:1775773761.016266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c707400. 00010000:00000010:2.1:1775773761.016267:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349aee00. 00010000:00000010:2.1:1775773761.016269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623441b800. 00010000:00000010:2.1:1775773761.016271:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349ae900. 00010000:00000010:2.1:1775773761.016272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623395e600. 00010000:00000010:2.1:1775773761.016274:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205044600. 00010000:00000010:2.1:1775773761.016275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c707000. 00010000:00000010:2.1:1775773761.016277:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349ae700. 00010000:00000010:2.1:1775773761.016279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623441a600. 00010000:00000010:2.1:1775773761.016280:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620adffa00. 00010000:00000010:2.1:1775773761.016282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623395fc00. 00010000:00000010:2.1:1775773761.016284:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfc8a00. 00010000:00000010:2.1:1775773761.016286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c706600. 00010000:00000010:2.1:1775773761.016298:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfc8100. 00010000:00000010:2.1:1775773761.016300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623441a000. 00010000:00000010:2.1:1775773761.016301:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfc8600. 00010000:00000010:2.1:1775773761.016303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623441aa00. 00010000:00000010:2.1:1775773761.016305:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620adff100. 00010000:00000010:2.1:1775773761.016306:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623395f800. 00010000:00000010:2.1:1775773761.016308:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620adff600. 00010000:00000010:2.1:1775773761.016309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62066db600. 00010000:00000010:2.1:1775773761.016311:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620adfff00. 00010000:00000010:2.1:1775773761.016312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623441b400. 00010000:00000010:2.1:1775773761.016314:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349aea00. 00010000:00000010:2.1:1775773761.016316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623441bc00. 00010000:00000010:2.1:1775773761.016317:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349ae100. 00010000:00000010:2.1:1775773761.016319:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623395ea00. 00010000:00000010:2.1:1775773761.016320:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349ae600. 00010000:00000010:2.1:1775773761.016322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623441ac00. 00010000:00000010:2.1:1775773761.016323:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349aef00. 00010000:00000010:2.1:1775773761.016325:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623441b600. 00010000:00000010:2.1:1775773761.016326:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349ae000. 00010000:00000010:2.1:1775773761.016328:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623441ae00. 00010000:00000010:2.1:1775773761.016329:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62349ae500. 00010000:00000010:2.1:1775773761.016331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623395f600. 00010000:00000010:2.1:1775773761.016332:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620adff000. 00010000:00000010:2.1:1775773761.016339:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623395ee00. 00010000:00000010:2.1:1775773761.016341:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfc8f00. 00010000:00000010:2.1:1775773761.016344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623395e200. 00010000:00000010:2.1:1775773761.016347:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ccece00. 00010000:00000010:2.1:1775773761.016348:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c707200. 00010000:00000010:2.1:1775773761.016350:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205044f00. 00010000:00000010:2.1:1775773761.016351:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623395fe00. 00010000:00000010:2.1:1775773761.016354:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bc9e900. 00010000:00000010:2.1:1775773761.016356:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623395f200. 00010000:00000010:2.1:1775773761.016357:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205044000. 00010000:00000010:2.1:1775773761.016359:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234998400. 00010000:00000010:2.1:1775773761.016361:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ccec900. 00010000:00000010:2.1:1775773761.016363:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62066dae00. 00010000:00000010:2.1:1775773761.016368:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bc9e700. 00010000:00000010:2.1:1775773761.016370:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c706200. 00010000:00000010:2.1:1775773761.016371:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bc9ea00. 00010000:00000010:2.1:1775773761.016373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234998600. 00010000:00000010:2.1:1775773761.016374:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bc9e100. 00010000:00000010:2.1:1775773761.016376:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234998000. 00010000:00000010:2.1:1775773761.016380:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bc9e600. 00010000:00000010:2.1:1775773761.016384:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c707600. 00010000:00000010:2.1:1775773761.016386:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ccec700. 00010000:00000010:2.1:1775773761.016388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623441ba00. 00010000:00000010:2.1:1775773761.016389:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cceca00. 00010000:00000010:2.1:1775773761.016391:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623441b000. 00010000:00000010:2.1:1775773761.016392:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ccec100. 00010000:00000010:2.1:1775773761.016394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623441be00. 00010000:00000010:2.1:1775773761.016395:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ccec600. 00010000:00000010:2.1:1775773761.016397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623441b200. 00010000:00000010:2.1:1775773761.016398:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205044500. 00010000:00000010:2.1:1775773761.016402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c706000. 00010000:00000010:2.1:1775773761.016406:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ccecf00. 00010000:00000010:2.1:1775773761.016408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623441a400. 00010000:00000010:2.1:1775773761.016409:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ccec000. 00010000:00000010:2.1:1775773761.016411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e62b400. 00010000:00000010:2.1:1775773761.016412:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ccec500. 00010000:00000010:2.1:1775773761.016416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e62bc00. 00000080:00000010:2.1:1775773771.252170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8080. 00000080:00000010:2.1:1775773771.252186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fb60. 00000080:00000010:2.1:1775773771.252191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fc60. 00000080:00000010:2.1:1775773771.256173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f060. 00000080:00000010:2.1:1775773771.256183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f7a0. 00000080:00000010:2.1:1775773771.256186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8ba0. 00000080:00000010:2.1:1775773771.256189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fb40. 00000080:00000010:2.1:1775773771.256192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371ff40. 00000080:00000010:2.1:1775773771.256194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f220. 00000080:00000010:2.1:1775773771.256197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f6e0. 00000080:00000010:2.1:1775773771.256202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f560. 00000080:00000010:2.1:1775773771.256205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8360. 00000080:00000010:2.1:1775773771.256208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0ff20. 00000080:00000010:2.1:1775773771.256213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd5c0. 00000080:00000010:2.1:1775773771.256217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371ffe0. 00000080:00000010:2.1:1775773771.256219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f500. 00000080:00000010:2.1:1775773771.256222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fba0. 00000080:00000010:2.1:1775773771.256224:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f280. 00000080:00000010:2.1:1775773771.256227:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd460. 00000080:00000010:2.1:1775773771.256232:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8e20. 00000080:00000010:2.1:1775773771.256235:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd7a0. 00000080:00000010:2.1:1775773771.256237:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd6c0. 00000080:00000010:2.1:1775773771.256240:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f100. 00000080:00000010:2.1:1775773771.256242:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f0c0. 00000080:00000010:2.1:1775773771.256245:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fc40. 00000080:00000010:2.1:1775773771.256249:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f5c0. 00000080:00000010:2.1:1775773771.256253:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdb40. 00000080:00000010:2.1:1775773771.256256:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8a60. 00000080:00000010:2.1:1775773771.256258:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f420. 00000080:00000010:2.1:1775773771.256262:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f960. 00000080:00000010:2.1:1775773771.256265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f6a0. 00000080:00000010:2.1:1775773771.256269:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fd00. 00000080:00000010:2.1:1775773771.256272:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8ee0. 00000080:00000010:2.1:1775773771.256274:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371ff00. 00000080:00000010:2.1:1775773771.256276:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fe80. 00000080:00000010:2.1:1775773771.256279:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a87a0. 00000080:00000010:2.1:1775773771.256281:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fbe0. 00000080:00000010:2.1:1775773771.256284:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fe20. 00000080:00000010:2.1:1775773771.256287:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdc60. 00000080:00000010:2.1:1775773771.256290:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fdc0. 00000080:00000010:2.1:1775773771.256292:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f300. 00000080:00000010:2.1:1775773771.256295:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f140. 00000080:00000010:2.1:1775773771.256297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd360. 00000080:00000010:2.1:1775773771.256300:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fa00. 00000080:00000010:2.1:1775773771.256303:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620a8bae40. 00000080:00000010:2.1:1775773771.256308:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd8e0. 00000080:00000010:2.1:1775773771.256311:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620a8ba620. 00000080:00000010:2.1:1775773771.256314:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fc80. 00000080:00000010:2.1:1775773771.256316:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8100. 00000080:00000010:2.1:1775773771.256319:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd740. 00000080:00000010:2.1:1775773771.256324:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620a8ba720. 00000080:00000010:2.1:1775773771.256327:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4b80. 00000080:00000010:2.1:1775773771.256332:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396200. 00000080:00000010:2.1:1775773771.256335:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdfa0. 00000080:00000010:2.1:1775773771.256338:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396ac0. 00000080:00000010:2.1:1775773771.256340:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396140. 00000080:00000010:2.1:1775773771.256343:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8060. 00000080:00000010:2.1:1775773771.256346:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b1b5b20. 00000080:00000010:2.1:1775773771.256349:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396dc0. 00000080:00000010:2.1:1775773771.256352:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396580. 00000080:00000010:2.1:1775773771.256354:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396120. 00000080:00000010:2.1:1775773771.256357:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396de0. 00000080:00000010:2.1:1775773771.256359:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396d60. 00000080:00000010:2.1:1775773771.256362:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396740. 00000080:00000010:2.1:1775773771.256364:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396ae0. 00000080:00000010:2.1:1775773771.256367:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396c80. 00000080:00000010:2.1:1775773771.256369:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f3a0. 00000080:00000010:2.1:1775773771.256372:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd7e0. 00000080:00000010:2.1:1775773771.256374:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396fa0. 00000080:00000010:2.1:1775773771.256377:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b1b5de0. 00000080:00000010:2.1:1775773771.256379:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b1b5f20. 00000080:00000010:2.1:1775773771.256381:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396b00. 00000080:00000010:2.1:1775773771.256384:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396da0. 00000080:00000010:2.1:1775773771.256386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e3967c0. 00000080:00000010:2.1:1775773771.256389:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396700. 00000080:00000010:2.1:1775773771.256391:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e3963a0. 00000080:00000010:2.1:1775773771.256394:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fda80. 00000080:00000010:2.1:1775773771.256396:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396260. 00000080:00000010:2.1:1775773771.256398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e3969a0. 00000080:00000010:2.1:1775773771.256403:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396f60. 00000080:00000010:2.1:1775773771.256405:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396bc0. 00000080:00000010:2.1:1775773771.256408:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e3960e0. 00000080:00000010:2.1:1775773771.256410:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396ca0. 00000080:00000010:2.1:1775773771.256413:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396b80. 00000080:00000010:2.1:1775773771.256415:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd260. 00000080:00000010:2.1:1775773771.256418:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057eb5c0. 00000080:00000010:2.1:1775773771.256421:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622cf93500. 00000080:00000010:2.1:1775773771.256424:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdec0. 00000080:00000010:2.1:1775773771.256426:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fe20. 00000080:00000010:2.1:1775773771.256429:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4600. 00000080:00000010:2.1:1775773771.256431:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4580. 00000080:00000010:2.1:1775773771.256433:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd880. 00000080:00000010:2.1:1775773771.256436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b1b5560. 00000080:00000010:2.1:1775773771.256438:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c505ac0. 00000080:00000010:2.1:1775773771.256441:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b1b52e0. 00000080:00000010:2.1:1775773771.256444:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210e5c740. 00000080:00000010:2.1:1775773771.256446:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa120. 00000080:00000010:2.1:1775773771.256449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd540. 00000080:00000010:2.1:1775773771.256453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04020. 00000080:00000010:2.1:1775773771.256456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6220367d00. 00000080:00000010:2.1:1775773771.256458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04b60. 00000080:00000010:2.1:1775773771.256461:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f1c0. 00000080:00000010:2.1:1775773771.256463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04b20. 00000080:00000010:2.1:1775773771.256466:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fc40. 00000080:00000010:2.1:1775773771.256469:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f780. 00000080:00000010:2.1:1775773771.256471:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f760. 00000080:00000010:2.1:1775773771.256473:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdfc0. 00000080:00000010:2.1:1775773771.256475:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f1a0. 00000080:00000010:2.1:1775773771.256478:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd480. 00000080:00000010:2.1:1775773771.256480:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fc00. 00000080:00000010:2.1:1775773771.256483:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f3e0. 00000080:00000010:2.1:1775773771.256485:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04d00. 00000080:00000010:2.1:1775773771.256488:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b042a0. 00000080:00000010:2.1:1775773771.256492:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b041e0. 00000080:00000010:2.1:1775773771.256494:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdba0. 00000080:00000010:2.1:1775773771.256496:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04320. 00000080:00000010:2.1:1775773771.256499:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd620. 00000080:00000010:2.1:1775773771.256501:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04480. 00000080:00000010:2.1:1775773771.256503:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04ae0. 00000080:00000010:2.1:1775773771.256506:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd2a0. 00000080:00000010:2.1:1775773771.256508:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fde80. 00000080:00000010:2.1:1775773771.256510:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04760. 00000080:00000010:2.1:1775773771.256512:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04da0. 00000080:00000010:2.1:1775773771.256515:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f600. 00000080:00000010:2.1:1775773771.256517:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04580. 00000080:00000010:2.1:1775773771.256519:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04c40. 00000080:00000010:2.1:1775773771.256522:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6220367220. 00000080:00000010:2.1:1775773771.256524:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210e5ca60. 00000080:00000010:2.1:1775773771.256527:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04540. 00000080:00000010:2.1:1775773771.256529:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04700. 00000080:00000010:2.1:1775773771.256531:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210e5c1e0. 00000080:00000010:2.1:1775773771.256533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210e5cdc0. 00000080:00000010:2.1:1775773771.256536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b049c0. 00000080:00000010:2.1:1775773771.256538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04380. 00000080:00000010:2.1:1775773771.256540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210e5c920. 00000080:00000010:2.1:1775773771.256542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04d20. 00000080:00000010:2.1:1775773771.256544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f260. 00000080:00000010:2.1:1775773771.256546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210e5c900. 00000080:00000010:2.1:1775773771.256549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdc80. 00000080:00000010:2.1:1775773771.256551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04ea0. 00000080:00000010:2.1:1775773771.256553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04840. 00000080:00000010:2.1:1775773771.256555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fd20. 00000080:00000010:2.1:1775773771.256557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04f60. 00000080:00000010:2.1:1775773771.256559:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210e5cd40. 00000080:00000010:2.1:1775773771.256562:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04940. 00000080:00000010:2.1:1775773771.256564:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04880. 00000080:00000010:2.1:1775773771.256567:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f9c0. 00000080:00000010:2.1:1775773771.256569:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd340. 00000080:00000010:2.1:1775773771.256571:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f9a0. 00000080:00000010:2.1:1775773771.256573:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fde40. 00000080:00000010:2.1:1775773771.256576:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fe40. 00000080:00000010:2.1:1775773771.256578:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6205086160. 00000080:00000010:2.1:1775773771.256580:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdf20. 00000080:00000010:2.1:1775773771.256583:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd500. 00000080:00000010:2.1:1775773771.256585:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fb60. 00000080:00000010:2.1:1775773771.256587:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd4c0. 00000080:00000010:2.1:1775773771.256589:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd920. 00000080:00000010:2.1:1775773771.256591:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04be0. 00000080:00000010:2.1:1775773771.256593:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225b04b40. 00000080:00000010:2.1:1775773771.256595:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db40e0. 00010000:00000010:2.1:1775773771.256599:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620add7c00. 00010000:00000010:2.1:1775773771.256602:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3e9a00. 00010000:00000010:2.1:1775773771.256648:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c069500. 00010000:00000010:2.1:1775773771.256650:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234799000. 00010000:00000010:2.1:1775773771.256652:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620add7300. 00010000:00000010:2.1:1775773771.256654:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234799e00. 00010000:00000010:2.1:1775773771.256656:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225977500. 00010000:00000010:2.1:1775773771.256657:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1dc00. 00010000:00000010:2.1:1775773771.256659:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620add7200. 00010000:00000010:2.1:1775773771.256662:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0fa00. 00010000:00000010:2.1:1775773771.256663:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225977b00. 00010000:00000010:2.1:1775773771.256665:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3e9000. 00010000:00000010:2.1:1775773771.256667:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c069b00. 00010000:00000010:2.1:1775773771.256669:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0e200. 00010000:00000010:2.1:1775773771.256671:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c069c00. 00010000:00000010:2.1:1775773771.256673:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0fe00. 00010000:00000010:2.1:1775773771.256674:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c069300. 00010000:00000010:2.1:1775773771.256676:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0f200. 00010000:00000010:2.1:1775773771.256679:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab16d00. 00010000:00000010:2.1:1775773771.256681:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1cc00. 00010000:00000010:2.1:1775773771.256683:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab16400. 00010000:00000010:2.1:1775773771.256685:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1de00. 00010000:00000010:2.1:1775773771.256687:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620add7d00. 00010000:00000010:2.1:1775773771.256689:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3e9e00. 00010000:00000010:2.1:1775773771.256691:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233713400. 00010000:00000010:2.1:1775773771.256692:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1d600. 00010000:00000010:2.1:1775773771.256694:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233713800. 00010000:00000010:2.1:1775773771.256696:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620af22a00. 00010000:00000010:2.1:1775773771.256698:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c069200. 00010000:00000010:2.1:1775773771.256700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0e400. 00010000:00000010:2.1:1775773771.256702:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c069d00. 00010000:00000010:2.1:1775773771.256703:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0e800. 00010000:00000010:2.1:1775773771.256705:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c069400. 00010000:00000010:2.1:1775773771.256706:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0f800. 00010000:00000010:2.1:1775773771.256708:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab16800. 00010000:00000010:2.1:1775773771.256710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1c200. 00010000:00000010:2.1:1775773771.256712:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620add7400. 00010000:00000010:2.1:1775773771.256714:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620af23800. 00010000:00000010:2.1:1775773771.256716:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620add7800. 00010000:00000010:2.1:1775773771.256718:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3e9200. 00010000:00000010:2.1:1775773771.256719:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c069800. 00010000:00000010:2.1:1775773771.256721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620af22400. 00010000:00000010:2.1:1775773771.256723:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233713e00. 00010000:00000010:2.1:1775773771.256724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620af23a00. 00010000:00000010:2.1:1775773771.256726:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c069e00. 00010000:00000010:2.1:1775773771.256728:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0e600. 00010000:00000010:2.1:1775773771.256729:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c069900. 00010000:00000010:2.1:1775773771.256731:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0e000. 00010000:00000010:2.1:1775773771.256733:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c069700. 00010000:00000010:2.1:1775773771.256734:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0ea00. 00010000:00000010:2.1:1775773771.256736:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c069a00. 00010000:00000010:2.1:1775773771.256738:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0f400. 00010000:00000010:2.1:1775773771.256740:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c069100. 00010000:00000010:2.1:1775773771.256742:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620af22000. 00010000:00000010:2.1:1775773771.256744:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107f8a00. 00010000:00000010:2.1:1775773771.256747:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3e8400. 00010000:00000010:2.1:1775773771.256748:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233713900. 00010000:00000010:2.1:1775773771.256750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620321bc00. 00010000:00000010:2.1:1775773771.256752:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c069600. 00010000:00000010:2.1:1775773771.256754:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0fc00. 00010000:00000010:2.1:1775773771.256755:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c069f00. 00010000:00000010:2.1:1775773771.256757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620321ac00. 00010000:00000010:2.1:1775773771.256759:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c069000. 00010000:00000010:2.1:1775773771.256761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0ec00. 00010000:00000010:2.1:1775773771.256762:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107f8100. 00010000:00000010:2.1:1775773771.256764:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3e8800. 00010000:00000010:2.1:1775773771.256766:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220263100. 00010000:00000010:2.1:1775773771.256768:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0f600. 00010000:00000010:2.1:1775773771.256770:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220263600. 00010000:00000010:2.1:1775773771.256771:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0ee00. 00010000:00000010:2.1:1775773771.256773:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107f8600. 00010000:00000010:2.1:1775773771.256775:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3e9800. 00010000:00000010:2.1:1775773771.256777:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220263f00. 00010000:00000010:2.1:1775773771.256779:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207d0f000. 00010000:00000010:2.1:1775773771.256781:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220263000. 00010000:00000010:2.1:1775773771.256783:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623499d200. 00010000:00000010:2.1:1775773771.256785:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233713700. 00010000:00000010:2.1:1775773771.256786:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620af22200. 00010000:00000010:2.1:1775773771.256788:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220263500. 00010000:00000010:2.1:1775773771.256790:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623499c400. 00010000:00000010:2.1:1775773771.256791:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220263b00. 00010000:00000010:2.1:1775773771.256793:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623499c800. 00010000:00000010:2.1:1775773771.256795:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220263c00. 00010000:00000010:2.1:1775773771.256797:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620321a000. 00010000:00000010:2.1:1775773771.256799:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220263300. 00010000:00000010:2.1:1775773771.256800:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620af23600. 00010000:00000010:2.1:1775773771.256803:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220263200. 00010000:00000010:2.1:1775773771.256804:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620321a800. 00010000:00000010:2.1:1775773771.256806:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233713a00. 00010000:00000010:2.1:1775773771.256808:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623499d800. 00010000:00000010:2.1:1775773771.256810:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cc21e00. 00010000:00000010:2.1:1775773771.256812:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620af23c00. 00010000:00000010:2.1:1775773771.256814:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107f8f00. 00010000:00000010:2.1:1775773771.256815:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623499c600. 00010000:00000010:2.1:1775773771.256817:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220263d00. 00010000:00000010:2.1:1775773771.256819:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620321ba00. 00010000:00000010:2.1:1775773771.256820:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107f8000. 00010000:00000010:2.1:1775773771.256822:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3e8600. 00010000:00000010:2.1:1775773771.256824:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220263400. 00010000:00000010:2.1:1775773771.256825:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620af22c00. 00010000:00000010:2.1:1775773771.256827:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220263800. 00010000:00000010:2.1:1775773771.256829:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623499c000. 00010000:00000010:2.1:1775773771.256830:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220263e00. 00010000:00000010:2.1:1775773771.256832:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623499ca00. 00010000:00000010:2.1:1775773771.256833:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220263900. 00010000:00000010:2.1:1775773771.256835:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623499d400. 00010000:00000010:2.1:1775773771.256837:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cc21900. 00010000:00000010:2.1:1775773771.256838:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620af22600. 00010000:00000010:2.1:1775773771.256840:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220263700. 00010000:00000010:2.1:1775773771.256842:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623499dc00. 00010000:00000010:2.1:1775773771.256843:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6220263a00. 00010000:00000010:2.1:1775773771.256845:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623499cc00. 00010000:00000010:2.1:1775773771.256846:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107f8500. 00010000:00000010:2.1:1775773771.256848:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3e8000. 00010000:00000010:2.1:1775773771.256850:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107f8b00. 00010000:00000010:2.1:1775773771.256852:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3e8a00. 00010000:00000010:2.1:1775773771.256855:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e3e5200. 00010000:00000010:2.1:1775773771.256857:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623499d600. 00010000:00000010:2.1:1775773771.256859:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab16e00. 00010000:00000010:2.1:1775773771.256860:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623499ce00. 00010000:00000010:2.1:1775773771.256862:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e3e5d00. 00010000:00000010:2.1:1775773771.256864:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623499c200. 00010000:00000010:2.1:1775773771.256866:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e3e5400. 00010000:00000010:2.1:1775773771.256867:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623499da00. 00010000:00000010:2.1:1775773771.256869:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107f8c00. 00010000:00000010:2.1:1775773771.256871:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623499d000. 00010000:00000010:2.1:1775773771.256872:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e3e5800. 00010000:00000010:2.1:1775773771.256874:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623499de00. 00010000:00000010:2.1:1775773771.256876:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e3e5e00. 00010000:00000010:2.1:1775773771.256878:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206630a00. 00010000:00000010:2.1:1775773771.256880:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e3e5900. 00010000:00000010:2.1:1775773771.256882:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206631400. 00010000:00000010:2.1:1775773771.256884:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e3e5700. 00010000:00000010:2.1:1775773771.256885:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620321aa00. 00010000:00000010:2.1:1775773771.256887:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cc21700. 00010000:00000010:2.1:1775773771.256889:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620af23000. 00010000:00000010:2.1:1775773771.256890:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e3e5a00. 00010000:00000010:2.1:1775773771.256892:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206631c00. 00010000:00000010:2.1:1775773771.256894:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107f8300. 00010000:00000010:2.1:1775773771.256896:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3e9400. 00010000:00000010:2.1:1775773771.256897:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107f8200. 00010000:00000010:2.1:1775773771.256899:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3e9c00. 00010000:00000010:2.1:1775773771.256900:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e3e5100. 00010000:00000010:2.1:1775773771.256902:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206630c00. 00010000:00000010:2.1:1775773771.256904:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e3e5600. 00010000:00000010:2.1:1775773771.256906:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206631600. 00010000:00000010:2.1:1775773771.256908:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e3e5f00. 00010000:00000010:2.1:1775773771.256910:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206630e00. 00010000:00000010:2.1:1775773771.256912:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e3e5000. 00010000:00000010:2.1:1775773771.256914:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206630200. 00010000:00000010:2.1:1775773771.256915:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107f8d00. 00010000:00000010:2.1:1775773771.256917:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206631a00. 00010000:00000010:2.1:1775773771.256919:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cc21a00. 00010000:00000010:2.1:1775773771.256921:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620af23200. 00010000:00000010:2.1:1775773771.256922:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e3e5500. 00010000:00000010:2.1:1775773771.256924:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206631000. 00010000:00000010:2.1:1775773771.256925:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e3e5b00. 00010000:00000010:2.1:1775773771.256927:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206631e00. 00010000:00000010:2.1:1775773771.256929:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e3e5c00. 00010000:00000010:2.1:1775773771.256930:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206631200. 00010000:00000010:2.1:1775773771.256932:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e3e5300. 00010000:00000010:2.1:1775773771.256933:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206630400. 00010000:00000010:2.1:1775773771.256935:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107f8400. 00010000:00000010:2.1:1775773771.256936:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206630800. 00010000:00000010:2.1:1775773771.256938:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e33c300. 00010000:00000010:2.1:1775773771.256940:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206631800. 00010000:00000010:2.1:1775773771.256942:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e33c200. 00010000:00000010:2.1:1775773771.256944:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206630600. 00010000:00000010:2.1:1775773771.256945:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cc21100. 00010000:00000010:2.1:1775773771.256947:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620af22e00. 00010000:00000010:2.1:1775773771.256949:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e33cd00. 00010000:00000010:2.1:1775773771.256950:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206630000. 00010000:00000010:2.1:1775773771.256952:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107f8800. 00010000:00000010:2.1:1775773771.256954:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211f13600. 00010000:00000010:2.1:1775773771.256956:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e33c400. 00010000:00000010:2.1:1775773771.256957:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226476800. 00010000:00000010:2.1:1775773771.256959:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107f8e00. 00010000:00000010:2.1:1775773771.256961:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620321ae00. 00010000:00000010:2.1:1775773771.256962:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e33c800. 00010000:00000010:2.1:1775773771.256965:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211f12e00. 00010000:00000010:2.1:1775773771.256967:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e33ce00. 00010000:00000010:2.1:1775773771.256969:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211f12200. 00010000:00000010:2.1:1775773771.256971:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cc21600. 00010000:00000010:2.1:1775773771.256973:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226476e00. 00010000:00000010:2.1:1775773771.256975:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107f8900. 00010000:00000010:2.1:1775773771.256976:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3e8c00. 00010000:00000010:2.1:1775773771.256978:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e33c900. 00010000:00000010:2.1:1775773771.256980:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211f13a00. 00010000:00000010:2.1:1775773771.256981:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62107f8700. 00010000:00000010:2.1:1775773771.256983:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3e9600. 00010000:00000010:2.1:1775773771.256985:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cc21f00. 00010000:00000010:2.1:1775773771.256986:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211f13000. 00010000:00000010:2.1:1775773771.256988:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e33c700. 00010000:00000010:2.1:1775773771.256990:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211f13e00. 00010000:00000010:2.1:1775773771.256991:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cc21000. 00010000:00000010:2.1:1775773771.256993:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226476000. 00010000:00000010:2.1:1775773771.256995:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e33ca00. 00010000:00000010:2.1:1775773771.256996:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211f13200. 00010000:00000010:2.1:1775773771.256998:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e33c100. 00010000:00000010:2.1:1775773771.256999:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211f12400. 00010000:00000010:2.1:1775773771.257001:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e33c600. 00010000:00000010:2.1:1775773771.257003:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211f12800. 00010000:00000010:2.1:1775773771.257004:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e33cf00. 00010000:00000010:2.1:1775773771.257006:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620321be00. 00010000:00000010:2.1:1775773771.257008:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e33c000. 00010000:00000010:2.1:1775773771.257009:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211f13800. 00010000:00000010:2.1:1775773771.257011:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab16900. 00010000:00000010:2.1:1775773771.257013:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620321b400. 00010000:00000010:2.1:1775773771.257014:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cc21500. 00010000:00000010:2.1:1775773771.257016:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620321b200. 00010000:00000010:2.1:1775773771.257017:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cc21b00. 00010000:00000010:2.1:1775773771.257020:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620321a400. 00010000:00000010:2.1:1775773771.257022:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cc21c00. 00010000:00000010:2.1:1775773771.257024:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226477a00. 00010000:00000010:2.1:1775773771.257026:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cc21300. 00010000:00000010:2.1:1775773771.257028:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620321b600. 00010000:00000010:2.1:1775773771.257029:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cc21200. 00010000:00000010:2.1:1775773771.257031:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226477200. 00010000:00000010:2.1:1775773771.257033:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623386cd00. 00010000:00000010:2.1:1775773771.257035:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620321a200. 00010000:00000010:2.1:1775773771.257036:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab16700. 00010000:00000010:2.1:1775773771.257038:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620321b000. 00010000:00000010:2.1:1775773771.257040:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623386c400. 00010000:00000010:2.1:1775773771.257041:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211f12600. 00010000:00000010:2.1:1775773771.257043:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e33c500. 00010000:00000010:2.1:1775773771.257044:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211f12000. 00010000:00000010:2.1:1775773771.257061:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e33cb00. 00010000:00000010:2.1:1775773771.257063:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211f12a00. 00010000:00000010:2.1:1775773771.257065:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab16a00. 00010000:00000010:2.1:1775773771.257067:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226476c00. 00010000:00000010:2.1:1775773771.257068:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e33cc00. 00010000:00000010:2.1:1775773771.257070:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211f13400. 00010000:00000010:2.1:1775773771.257072:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e524f00. 00010000:00000010:2.1:1775773771.257074:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226476a00. 00010000:00000010:2.1:1775773771.257076:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e524000. 00010000:00000010:2.1:1775773771.257077:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211f13c00. 00010000:00000010:2.1:1775773771.257079:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e524500. 00010000:00000010:2.1:1775773771.257081:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211f12c00. 00010000:00000010:2.1:1775773771.257083:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e524b00. 00010000:00000010:2.1:1775773771.257085:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226477c00. 00010000:00000010:2.1:1775773771.257087:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e524c00. 00010000:00000010:2.1:1775773771.257089:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226477600. 00010000:00000010:2.1:1775773771.257090:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e524300. 00010000:00000010:2.1:1775773771.257093:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c552400. 00010000:00000010:2.1:1775773771.257095:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e524200. 00010000:00000010:2.1:1775773771.257097:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c552800. 00010000:00000010:2.1:1775773771.257099:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab16100. 00010000:00000010:2.1:1775773771.257100:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620321a600. 00010000:00000010:2.1:1775773771.257102:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e524d00. 00010000:00000010:2.1:1775773771.257104:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c553800. 00010000:00000010:2.1:1775773771.257106:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e524400. 00010000:00000010:2.1:1775773771.257107:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c552600. 00010000:00000010:2.1:1775773771.257109:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e524800. 00010000:00000010:2.1:1775773771.257110:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226477000. 00010000:00000010:2.1:1775773771.257112:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623386c800. 00010000:00000010:2.1:1775773771.257114:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3e8e00. 00010000:00000010:2.1:1775773771.257115:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623386ce00. 00010000:00000010:2.1:1775773771.257117:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e3e8200. 00010000:00000010:2.1:1775773771.257119:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e524e00. 00010000:00000010:2.1:1775773771.257120:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c552000. 00010000:00000010:2.1:1775773771.257122:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e524900. 00010000:00000010:2.1:1775773771.257123:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c552a00. 00010000:00000010:2.1:1775773771.257125:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623386c900. 00010000:00000010:2.1:1775773771.257127:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bfbd400. 00010000:00000010:2.1:1775773771.257128:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623386c700. 00010000:00000010:2.1:1775773771.257130:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bfbdc00. 00010000:00000010:2.1:1775773771.257132:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e524700. 00010000:00000010:2.1:1775773771.257146:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c553400. 00010000:00000010:2.1:1775773771.257148:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e524a00. 00010000:00000010:2.1:1775773771.257149:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c553c00. 00010000:00000010:2.1:1775773771.257151:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623386ca00. 00010000:00000010:2.1:1775773771.257153:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bfbcc00. 00010000:00000010:2.1:1775773771.257154:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e524100. 00010000:00000010:2.1:1775773771.257156:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c552c00. 00010000:00000010:2.1:1775773771.257157:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cc21d00. 00010000:00000010:2.1:1775773771.257160:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620321b800. 00010000:00000010:2.1:1775773771.257162:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623386c100. 00010000:00000010:2.1:1775773771.257163:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bfbd600. 00010000:00000010:2.1:1775773771.257165:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cc21400. 00010000:00000010:2.1:1775773771.257167:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226477e00. 00010000:00000010:2.1:1775773771.257169:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e524600. 00010000:00000010:2.1:1775773771.257170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c553600. 00010000:00000010:2.1:1775773771.257172:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225922600. 00010000:00000010:2.1:1775773771.257174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c552e00. 00010000:00000010:2.1:1775773771.257175:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab16600. 00010000:00000010:2.1:1775773771.257177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226684600. 00010000:00000010:2.1:1775773771.257179:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225922f00. 00010000:00000010:2.1:1775773771.257180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c552200. 00010000:00000010:2.1:1775773771.257182:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623386c600. 00010000:00000010:2.1:1775773771.257183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bfbce00. 00010000:00000010:2.1:1775773771.257185:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623386cf00. 00010000:00000010:2.1:1775773771.257186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c553a00. 00010000:00000010:2.1:1775773771.257188:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225922000. 00010000:00000010:2.1:1775773771.257190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c553000. 00010000:00000010:2.1:1775773771.257191:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab16f00. 00010000:00000010:2.1:1775773771.257193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226685e00. 00010000:00000010:2.1:1775773771.257195:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab16000. 00010000:00000010:2.1:1775773771.257196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226476200. 00010000:00000010:2.1:1775773771.257198:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225922500. 00010000:00000010:2.1:1775773771.257199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226684000. 00010000:00000010:2.1:1775773771.257201:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab16500. 00010000:00000010:2.1:1775773771.257203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226477800. 00010000:00000010:2.1:1775773771.257204:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cc21800. 00010000:00000010:2.1:1775773771.257206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226685200. 00010000:00000010:2.1:1775773771.257207:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623386c000. 00010000:00000010:2.1:1775773771.257209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bfbc200. 00010000:00000010:2.1:1775773771.257215:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623386c500. 00010000:00000010:2.1:1775773771.257218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226476400. 00010000:00000010:2.1:1775773771.257219:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623386cb00. 00010000:00000010:2.1:1775773771.257225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226476600. 00010000:00000010:2.1:1775773771.257228:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab16b00. 00010000:00000010:2.1:1775773771.257229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226684400. 00010000:00000010:2.1:1775773771.257234:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab16c00. 00010000:00000010:2.1:1775773771.257236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226477400. 00010000:00000010:2.1:1775773771.257237:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab16300. 00010000:00000010:2.1:1775773771.257241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cfa4600. 00010000:00000010:2.1:1775773771.257246:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226601c00. 00010000:00000010:2.1:1775773771.257252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c553e00. 00010000:00000010:2.1:1775773771.257254:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225922b00. 00010000:00000010:2.1:1775773771.257258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c553200. 00000080:00000010:3.1:1775773780.457174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84560. 00000080:00000010:3.1:1775773780.461185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd849c0. 00000080:00000010:3.1:1775773780.461201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdb80. 00000080:00000010:3.1:1775773780.461208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84180. 00000080:00000010:3.1:1775773780.461217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd9e0. 00000080:00000010:3.1:1775773780.461225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd300. 00000080:00000010:3.1:1775773780.461232:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84a40. 00000080:00000010:3.1:1775773780.461238:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd842c0. 00000080:00000010:3.1:1775773780.461244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd846a0. 00000080:00000010:3.1:1775773780.461249:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd849e0. 00000080:00000010:3.1:1775773780.461255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdca0. 00000080:00000010:3.1:1775773780.461261:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb8a0. 00000080:00000010:3.1:1775773780.461270:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8760. 00000080:00000010:3.1:1775773780.461277:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84080. 00000080:00000010:3.1:1775773780.461286:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84ee0. 00000080:00000010:3.1:1775773780.461290:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb200. 00000080:00000010:3.1:1775773780.461294:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb060. 00000080:00000010:3.1:1775773780.461298:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84960. 00000080:00000010:3.1:1775773780.461303:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdce0. 00000080:00000010:3.1:1775773780.461309:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbec0. 00000080:00000010:3.1:1775773780.461314:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd840c0. 00000080:00000010:3.1:1775773780.461318:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbba0. 00000080:00000010:3.1:1775773780.461323:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8d00. 00000080:00000010:3.1:1775773780.461327:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdb00. 00000080:00000010:3.1:1775773780.461331:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a82a0. 00000080:00000010:3.1:1775773780.461335:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd842a0. 00000080:00000010:3.1:1775773780.461339:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84bc0. 00000080:00000010:3.1:1775773780.461342:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84620. 00000080:00000010:3.1:1775773780.461346:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fde00. 00000080:00000010:3.1:1775773780.461357:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbc80. 00000080:00000010:3.1:1775773780.461363:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84220. 00000080:00000010:3.1:1775773780.461369:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84ac0. 00000080:00000010:3.1:1775773780.461375:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84700. 00000080:00000010:3.1:1775773780.461378:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84be0. 00000080:00000010:3.1:1775773780.461382:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84800. 00000080:00000010:3.1:1775773780.461387:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84c40. 00000080:00000010:3.1:1775773780.461392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84e00. 00000080:00000010:3.1:1775773780.461396:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84b80. 00000080:00000010:3.1:1775773780.461400:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8aa0. 00000080:00000010:3.1:1775773780.461405:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233763d60. 00000080:00000010:3.1:1775773780.461409:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84c60. 00000080:00000010:3.1:1775773780.461413:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84f80. 00000080:00000010:3.1:1775773780.461417:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84d00. 00000080:00000010:3.1:1775773780.461420:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84ec0. 00000080:00000010:3.1:1775773780.461424:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84980. 00000080:00000010:3.1:1775773780.461428:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84b40. 00000080:00000010:3.1:1775773780.461430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84e80. 00000080:00000010:3.1:1775773780.461433:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c6cc560. 00000080:00000010:3.1:1775773780.461436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207dccaa0. 00000080:00000010:3.1:1775773780.461439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396080. 00000080:00000010:3.1:1775773780.461443:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6200025a40. 00000080:00000010:3.1:1775773780.461446:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8000. 00000080:00000010:3.1:1775773780.461449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd520. 00000080:00000010:3.1:1775773780.461452:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd126c0. 00000080:00000010:3.1:1775773780.461456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6200025980. 00000080:00000010:3.1:1775773780.461458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd980. 00000080:00000010:3.1:1775773780.461461:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620130b780. 00000080:00000010:3.1:1775773780.461464:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a85e0. 00000080:00000010:3.1:1775773780.461466:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd4e0. 00000080:00000010:3.1:1775773780.461469:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd020. 00000080:00000010:3.1:1775773780.461471:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6200025d40. 00000080:00000010:3.1:1775773780.461474:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8500. 00000080:00000010:3.1:1775773780.461476:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207dcc4c0. 00000080:00000010:3.1:1775773780.461479:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207dccc20. 00000080:00000010:3.1:1775773780.461481:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207dcc740. 00000080:00000010:3.1:1775773780.461483:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207dccc40. 00000080:00000010:3.1:1775773780.461485:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207dcc260. 00000080:00000010:3.1:1775773780.461488:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62000253a0. 00000080:00000010:3.1:1775773780.461490:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e2faec0. 00000080:00000010:3.1:1775773780.461494:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62183957a0. 00000080:00000010:3.1:1775773780.461497:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202a79ac0. 00000080:00000010:3.1:1775773780.461499:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6218395340. 00000080:00000010:3.1:1775773780.461502:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6218395300. 00000080:00000010:3.1:1775773780.461505:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62183956e0. 00000080:00000010:3.1:1775773780.461508:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6218395f20. 00000080:00000010:3.1:1775773780.461510:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6218395fe0. 00000080:00000010:3.1:1775773780.461517:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6218395020. 00000080:00000010:3.1:1775773780.461520:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6218395940. 00000080:00000010:3.1:1775773780.461522:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6200025e20. 00000080:00000010:3.1:1775773780.461525:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62183952e0. 00000080:00000010:3.1:1775773780.461527:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62183959e0. 00000080:00000010:3.1:1775773780.461529:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6218395320. 00000080:00000010:3.1:1775773780.461531:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6218395a60. 00000080:00000010:3.1:1775773780.461534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6200025b20. 00000080:00000010:3.1:1775773780.461536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202a79a40. 00000080:00000010:3.1:1775773780.461540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620130b180. 00000080:00000010:3.1:1775773780.461543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620a8ba320. 00000080:00000010:3.1:1775773780.461545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e3960c0. 00000080:00000010:3.1:1775773780.461548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f000. 00000080:00000010:3.1:1775773780.461551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620130bcc0. 00000080:00000010:3.1:1775773780.461554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62000259c0. 00000080:00000010:3.1:1775773780.461556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6200025aa0. 00000080:00000010:3.1:1775773780.461559:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62000259e0. 00000080:00000010:3.1:1775773780.461561:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f520. 00000080:00000010:3.1:1775773780.461564:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620130be60. 00000080:00000010:3.1:1775773780.461566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620130bbe0. 00000080:00000010:3.1:1775773780.461568:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd3c0. 00000080:00000010:3.1:1775773780.461571:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6200025a60. 00000080:00000010:3.1:1775773780.461573:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0fec0. 00000080:00000010:3.1:1775773780.461575:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62000259a0. 00000080:00000010:3.1:1775773780.461577:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0f360. 00000080:00000010:3.1:1775773780.461580:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e396280. 00000080:00000010:3.1:1775773780.461582:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a82e0. 00000080:00000010:3.1:1775773780.461585:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620130b860. 00000080:00000010:3.1:1775773780.461587:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd320. 00000080:00000010:3.1:1775773780.461589:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b1689e0. 00000080:00000010:3.1:1775773780.461592:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd12580. 00000080:00000010:3.1:1775773780.461595:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233edb820. 00000080:00000010:3.1:1775773780.461597:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622c6cc440. 00000080:00000010:3.1:1775773780.461600:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620aae2140. 00000080:00000010:3.1:1775773780.461602:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210e5c480. 00000080:00000010:3.1:1775773780.461605:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f040. 00000080:00000010:3.1:1775773780.461608:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f4a0. 00000080:00000010:3.1:1775773780.461610:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdac0. 00000080:00000010:3.1:1775773780.461613:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd120. 00000080:00000010:3.1:1775773780.461615:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f660. 00000080:00000010:3.1:1775773780.461617:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdcc0. 00000080:00000010:3.1:1775773780.461620:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620130b8e0. 00000080:00000010:3.1:1775773780.461622:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f880. 00000080:00000010:3.1:1775773780.461625:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f3e0. 00000080:00000010:3.1:1775773780.461627:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f580. 00000080:00000010:3.1:1775773780.461629:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620130bec0. 00000080:00000010:3.1:1775773780.461631:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f8e0. 00000080:00000010:3.1:1775773780.461634:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620792c8a0. 00000080:00000010:3.1:1775773780.461636:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fda20. 00000080:00000010:3.1:1775773780.461638:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd560. 00000080:00000010:3.1:1775773780.461640:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdb20. 00000080:00000010:3.1:1775773780.461643:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f1c0. 00000080:00000010:3.1:1775773780.461645:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a81c0. 00000080:00000010:3.1:1775773780.461648:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fd80. 00000080:00000010:3.1:1775773780.461650:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371f9c0. 00000080:00000010:3.1:1775773780.461652:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fb00. 00000080:00000010:3.1:1775773780.461654:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c0ff80. 00000080:00000010:3.1:1775773780.461656:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdfe0. 00000080:00000010:3.1:1775773780.461658:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f623371fa60. 00000080:00000010:3.1:1775773780.461660:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fda40. 00000080:00000010:3.1:1775773780.461662:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd800. 00010000:00000010:3.1:1775773780.461666:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225977300. 00010000:00000010:3.1:1775773780.461669:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e103800. 00010000:00000010:3.1:1775773780.461671:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233754700. 00010000:00000010:3.1:1775773780.461673:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e102600. 00010000:00000010:3.1:1775773780.461675:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233754a00. 00010000:00000010:3.1:1775773780.461677:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226471c00. 00010000:00000010:3.1:1775773780.461679:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233754100. 00010000:00000010:3.1:1775773780.461681:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e102000. 00010000:00000010:3.1:1775773780.461683:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62057d4b00. 00010000:00000010:3.1:1775773780.461685:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226471800. 00010000:00000010:3.1:1775773780.461687:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62057d4c00. 00010000:00000010:3.1:1775773780.461689:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c934800. 00010000:00000010:3.1:1775773780.461691:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233754600. 00010000:00000010:3.1:1775773780.461693:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e102a00. 00010000:00000010:3.1:1775773780.461694:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233754f00. 00010000:00000010:3.1:1775773780.461696:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e103400. 00010000:00000010:3.1:1775773780.461698:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62057d4300. 00010000:00000010:3.1:1775773780.461699:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e103c00. 00010000:00000010:3.1:1775773780.461701:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233754000. 00010000:00000010:3.1:1775773780.461703:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e102c00. 00010000:00000010:3.1:1775773780.461704:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62057d4200. 00010000:00000010:3.1:1775773780.461706:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c934e00. 00010000:00000010:3.1:1775773780.461709:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202ca300. 00010000:00000010:3.1:1775773780.461712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623364b000. 00010000:00000010:3.1:1775773780.461714:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e122c00. 00010000:00000010:3.1:1775773780.461717:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225805800. 00010000:00000010:3.1:1775773780.461719:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202ca200. 00010000:00000010:3.1:1775773780.461721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e103600. 00010000:00000010:3.1:1775773780.461723:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202cad00. 00010000:00000010:3.1:1775773780.461725:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233a99e00. 00010000:00000010:3.1:1775773780.461727:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225977200. 00010000:00000010:3.1:1775773780.461729:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623364ac00. 00010000:00000010:3.1:1775773780.461731:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225977d00. 00010000:00000010:3.1:1775773780.461732:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623364a600. 00010000:00000010:3.1:1775773780.461734:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202ca400. 00010000:00000010:3.1:1775773780.461736:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233a99200. 00010000:00000010:3.1:1775773780.461737:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62057d4d00. 00010000:00000010:3.1:1775773780.461739:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c935200. 00010000:00000010:3.1:1775773780.461741:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62057d4400. 00010000:00000010:3.1:1775773780.461743:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623364bc00. 00010000:00000010:3.1:1775773780.461744:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202ca800. 00010000:00000010:3.1:1775773780.461746:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233a98400. 00010000:00000010:3.1:1775773780.461748:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202cae00. 00010000:00000010:3.1:1775773780.461750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623364be00. 00010000:00000010:3.1:1775773780.461751:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e122300. 00010000:00000010:3.1:1775773780.461753:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225804600. 00010000:00000010:3.1:1775773780.461755:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202ca900. 00010000:00000010:3.1:1775773780.461757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c934000. 00010000:00000010:3.1:1775773780.461758:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202ca700. 00010000:00000010:3.1:1775773780.461760:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225804000. 00010000:00000010:3.1:1775773780.461762:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202caa00. 00010000:00000010:3.1:1775773780.461764:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233a98800. 00010000:00000010:3.1:1775773780.461766:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e122200. 00010000:00000010:3.1:1775773780.461768:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233a99800. 00010000:00000010:3.1:1775773780.461771:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202ca100. 00010000:00000010:3.1:1775773780.461772:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233a98600. 00010000:00000010:3.1:1775773780.461774:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62057d4800. 00010000:00000010:3.1:1775773780.461776:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c935000. 00010000:00000010:3.1:1775773780.461778:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62057d4e00. 00010000:00000010:3.1:1775773780.461780:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623364a200. 00010000:00000010:3.1:1775773780.461782:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202ca600. 00010000:00000010:3.1:1775773780.461783:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233a98000. 00010000:00000010:3.1:1775773780.461785:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202caf00. 00010000:00000010:3.1:1775773780.461787:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233a98a00. 00010000:00000010:3.1:1775773780.461788:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202ca000. 00010000:00000010:3.1:1775773780.461790:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233a99400. 00010000:00000010:3.1:1775773780.461791:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202ca500. 00010000:00000010:3.1:1775773780.461793:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233a99c00. 00010000:00000010:3.1:1775773780.461794:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225977400. 00010000:00000010:3.1:1775773780.461796:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233a98c00. 00010000:00000010:3.1:1775773780.461798:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202cab00. 00010000:00000010:3.1:1775773780.461799:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233a99600. 00010000:00000010:3.1:1775773780.461801:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202cac00. 00010000:00000010:3.1:1775773780.461802:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233a98e00. 00010000:00000010:3.1:1775773780.461804:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd0e00. 00010000:00000010:3.1:1775773780.461806:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233a98200. 00010000:00000010:3.1:1775773780.461808:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd0900. 00010000:00000010:3.1:1775773780.461810:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233a99a00. 00010000:00000010:3.1:1775773780.461811:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225977800. 00010000:00000010:3.1:1775773780.461813:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623364a800. 00010000:00000010:3.1:1775773780.461815:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd0700. 00010000:00000010:3.1:1775773780.461816:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233a99000. 00010000:00000010:3.1:1775773780.461818:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd0a00. 00010000:00000010:3.1:1775773780.461820:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233eb4a00. 00010000:00000010:3.1:1775773780.461822:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e122d00. 00010000:00000010:3.1:1775773780.461824:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233eb5400. 00010000:00000010:3.1:1775773780.461827:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd0100. 00010000:00000010:3.1:1775773780.461829:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233eb5c00. 00010000:00000010:3.1:1775773780.461831:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd0600. 00010000:00000010:3.1:1775773780.461833:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233eb4c00. 00010000:00000010:3.1:1775773780.461834:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd0f00. 00010000:00000010:3.1:1775773780.461836:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233eb5600. 00010000:00000010:3.1:1775773780.461838:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd0000. 00010000:00000010:3.1:1775773780.461840:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233eb4e00. 00010000:00000010:3.1:1775773780.461841:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd0500. 00010000:00000010:3.1:1775773780.461843:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233eb4200. 00010000:00000010:3.1:1775773780.461844:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd0b00. 00010000:00000010:3.1:1775773780.461846:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233eb5a00. 00010000:00000010:3.1:1775773780.461848:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd0c00. 00010000:00000010:3.1:1775773780.461849:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233eb5000. 00010000:00000010:3.1:1775773780.461851:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd0300. 00010000:00000010:3.1:1775773780.461852:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233eb5e00. 00010000:00000010:3.1:1775773780.461854:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e122400. 00010000:00000010:3.1:1775773780.461856:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225804a00. 00010000:00000010:3.1:1775773780.461857:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62057d4900. 00010000:00000010:3.1:1775773780.461859:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c935c00. 00010000:00000010:3.1:1775773780.461861:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e122800. 00010000:00000010:3.1:1775773780.461863:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623364b200. 00010000:00000010:3.1:1775773780.461864:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e122e00. 00010000:00000010:3.1:1775773780.461866:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623364a400. 00010000:00000010:3.1:1775773780.461867:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62057d4700. 00010000:00000010:3.1:1775773780.461869:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c935800. 00010000:00000010:3.1:1775773780.461870:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225977900. 00010000:00000010:3.1:1775773780.461872:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233eb5200. 00010000:00000010:3.1:1775773780.461874:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd0200. 00010000:00000010:3.1:1775773780.461875:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233eb4400. 00010000:00000010:3.1:1775773780.461877:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233713100. 00010000:00000010:3.1:1775773780.461879:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c934a00. 00010000:00000010:3.1:1775773780.461881:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233713600. 00010000:00000010:3.1:1775773780.461883:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c934400. 00010000:00000010:3.1:1775773780.461898:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd0d00. 00010000:00000010:3.1:1775773780.461900:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623364aa00. 00010000:00000010:3.1:1775773780.461902:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd0400. 00010000:00000010:3.1:1775773780.461904:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233eb4800. 00010000:00000010:3.1:1775773780.461905:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e122900. 00010000:00000010:3.1:1775773780.461907:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623364a000. 00010000:00000010:3.1:1775773780.461909:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202dd0800. 00010000:00000010:3.1:1775773780.461911:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233eb5800. 00010000:00000010:3.1:1775773780.461913:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e187a00. 00010000:00000010:3.1:1775773780.461915:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233eb4600. 00010000:00000010:3.1:1775773780.461917:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e187100. 00010000:00000010:3.1:1775773780.461918:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233eb4000. 00010000:00000010:3.1:1775773780.461920:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e122700. 00010000:00000010:3.1:1775773780.461922:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb5f400. 00010000:00000010:3.1:1775773780.461924:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e187600. 00010000:00000010:3.1:1775773780.461926:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb5fc00. 00010000:00000010:3.1:1775773780.461929:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620add7e00. 00010000:00000010:3.1:1775773780.461931:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623364ba00. 00010000:00000010:3.1:1775773780.461932:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e187f00. 00010000:00000010:3.1:1775773780.461934:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb5ec00. 00010000:00000010:3.1:1775773780.461936:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e187000. 00010000:00000010:3.1:1775773780.461938:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb5f600. 00010000:00000010:3.1:1775773780.461939:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620add7900. 00010000:00000010:3.1:1775773780.461941:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb5ee00. 00010000:00000010:3.1:1775773780.461943:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e187500. 00010000:00000010:3.1:1775773780.461944:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb5e200. 00010000:00000010:3.1:1775773780.461946:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e187b00. 00010000:00000010:3.1:1775773780.461947:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb5fa00. 00010000:00000010:3.1:1775773780.461949:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e187c00. 00010000:00000010:3.1:1775773780.461951:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb5f000. 00010000:00000010:3.1:1775773780.461953:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e187300. 00010000:00000010:3.1:1775773780.461955:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb5fe00. 00010000:00000010:3.1:1775773780.461956:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e187200. 00010000:00000010:3.1:1775773780.461958:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb5f200. 00010000:00000010:3.1:1775773780.461960:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e187d00. 00010000:00000010:3.1:1775773780.461962:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb5e400. 00010000:00000010:3.1:1775773780.461963:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e187400. 00010000:00000010:3.1:1775773780.461965:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb5e800. 00010000:00000010:3.1:1775773780.461966:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620add7700. 00010000:00000010:3.1:1775773780.461968:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623364ae00. 00010000:00000010:3.1:1775773780.461970:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e187800. 00010000:00000010:3.1:1775773780.461971:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb5f800. 00010000:00000010:3.1:1775773780.461973:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e187e00. 00010000:00000010:3.1:1775773780.461974:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb5e600. 00010000:00000010:3.1:1775773780.461976:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e187900. 00010000:00000010:3.1:1775773780.461977:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb5e000. 00010000:00000010:3.1:1775773780.461979:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e187700. 00010000:00000010:3.1:1775773780.461980:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cb5ea00. 00010000:00000010:3.1:1775773780.461982:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e177e00. 00010000:00000010:3.1:1775773780.461985:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1d000. 00010000:00000010:3.1:1775773780.461986:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e177900. 00010000:00000010:3.1:1775773780.461988:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45200. 00010000:00000010:3.1:1775773780.461990:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620add7a00. 00010000:00000010:3.1:1775773780.461992:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44400. 00010000:00000010:3.1:1775773780.461994:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e177700. 00010000:00000010:3.1:1775773780.461996:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44800. 00010000:00000010:3.1:1775773780.461998:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e177a00. 00010000:00000010:3.1:1775773780.461999:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45800. 00010000:00000010:3.1:1775773780.462001:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e177100. 00010000:00000010:3.1:1775773780.462003:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44600. 00010000:00000010:3.1:1775773780.462004:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e177600. 00010000:00000010:3.1:1775773780.462006:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44000. 00010000:00000010:3.1:1775773780.462008:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233713f00. 00010000:00000010:3.1:1775773780.462010:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1d400. 00010000:00000010:3.1:1775773780.462012:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620add7100. 00010000:00000010:3.1:1775773780.462014:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1c800. 00010000:00000010:3.1:1775773780.462016:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225977700. 00010000:00000010:3.1:1775773780.462017:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1da00. 00010000:00000010:3.1:1775773780.462019:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e177f00. 00010000:00000010:3.1:1775773780.462021:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44a00. 00010000:00000010:3.1:1775773780.462022:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e177000. 00010000:00000010:3.1:1775773780.462024:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45400. 00010000:00000010:3.1:1775773780.462025:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e177500. 00010000:00000010:3.1:1775773780.462027:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45c00. 00010000:00000010:3.1:1775773780.462029:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233713000. 00010000:00000010:3.1:1775773780.462030:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c935400. 00010000:00000010:3.1:1775773780.462032:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233713500. 00010000:00000010:3.1:1775773780.462034:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1c600. 00010000:00000010:3.1:1775773780.462035:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e177b00. 00010000:00000010:3.1:1775773780.462037:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44c00. 00010000:00000010:3.1:1775773780.462038:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225977a00. 00010000:00000010:3.1:1775773780.462040:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1d800. 00010000:00000010:3.1:1775773780.462042:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e177c00. 00010000:00000010:3.1:1775773780.462044:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45600. 00010000:00000010:3.1:1775773780.462045:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e177300. 00010000:00000010:3.1:1775773780.462047:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44e00. 00010000:00000010:3.1:1775773780.462049:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620add7600. 00010000:00000010:3.1:1775773780.462050:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225805400. 00010000:00000010:3.1:1775773780.462052:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e177200. 00010000:00000010:3.1:1775773780.462053:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1c000. 00010000:00000010:3.1:1775773780.462055:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233713b00. 00010000:00000010:3.1:1775773780.462057:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c934600. 00010000:00000010:3.1:1775773780.462058:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e177d00. 00010000:00000010:3.1:1775773780.462060:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce44200. 00010000:00000010:3.1:1775773780.462062:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e177400. 00010000:00000010:3.1:1775773780.462064:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45a00. 00010000:00000010:3.1:1775773780.462066:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e177800. 00010000:00000010:3.1:1775773780.462067:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45000. 00010000:00000010:3.1:1775773780.462069:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620add7f00. 00010000:00000010:3.1:1775773780.462071:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ce45e00. 00010000:00000010:3.1:1775773780.462073:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc0cc00. 00010000:00000010:3.1:1775773780.462076:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234799200. 00010000:00000010:3.1:1775773780.462078:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc0c300. 00010000:00000010:3.1:1775773780.462080:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234798400. 00010000:00000010:3.1:1775773780.462081:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225977100. 00010000:00000010:3.1:1775773780.462083:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234798800. 00010000:00000010:3.1:1775773780.462085:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc0c200. 00010000:00000010:3.1:1775773780.462087:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234799800. 00010000:00000010:3.1:1775773780.462088:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc0cd00. 00010000:00000010:3.1:1775773780.462090:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c935e00. 00010000:00000010:3.1:1775773780.462092:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225977600. 00010000:00000010:3.1:1775773780.462094:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c934200. 00010000:00000010:3.1:1775773780.462096:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc0c400. 00010000:00000010:3.1:1775773780.462097:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234798600. 00010000:00000010:3.1:1775773780.462099:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233713c00. 00010000:00000010:3.1:1775773780.462101:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c934c00. 00010000:00000010:3.1:1775773780.462102:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc0c800. 00010000:00000010:3.1:1775773780.462104:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1ce00. 00010000:00000010:3.1:1775773780.462105:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc0ce00. 00010000:00000010:3.1:1775773780.462107:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234798000. 00010000:00000010:3.1:1775773780.462109:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc0c900. 00010000:00000010:3.1:1775773780.462110:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234798a00. 00010000:00000010:3.1:1775773780.462112:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc0c700. 00010000:00000010:3.1:1775773780.462114:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234799400. 00010000:00000010:3.1:1775773780.462115:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc0ca00. 00010000:00000010:3.1:1775773780.462117:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1ca00. 00010000:00000010:3.1:1775773780.462119:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc0c100. 00010000:00000010:3.1:1775773780.462120:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234799c00. 00010000:00000010:3.1:1775773780.462122:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620add7000. 00010000:00000010:3.1:1775773780.462124:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1d200. 00010000:00000010:3.1:1775773780.462126:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233713300. 00010000:00000010:3.1:1775773780.462127:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c935600. 00010000:00000010:3.1:1775773780.462129:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233713200. 00010000:00000010:3.1:1775773780.462130:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c935a00. 00010000:00000010:3.1:1775773780.462132:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233713d00. 00010000:00000010:3.1:1775773780.462147:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620af23400. 00010000:00000010:3.1:1775773780.462149:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc0c600. 00010000:00000010:3.1:1775773780.462151:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234798c00. 00010000:00000010:3.1:1775773780.462152:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620add7500. 00010000:00000010:3.1:1775773780.462154:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225805c00. 00010000:00000010:3.1:1775773780.462156:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc0cf00. 00010000:00000010:3.1:1775773780.462157:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234799600. 00010000:00000010:3.1:1775773780.462159:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc0c000. 00010000:00000010:3.1:1775773780.462161:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234798e00. 00010000:00000010:3.1:1775773780.462162:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc0c500. 00010000:00000010:3.1:1775773780.462164:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234798200. 00010000:00000010:3.1:1775773780.462166:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225977f00. 00010000:00000010:3.1:1775773780.462167:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207b1c400. 00010000:00000010:3.1:1775773780.462169:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225977000. 00010000:00000010:3.1:1775773780.462170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620af22800. 00010000:00000010:3.1:1775773780.462172:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dc0cb00. 00010000:00000010:3.1:1775773780.462173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234799a00. 00010000:00000010:3.1:1775773780.462175:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620add7b00. 00010000:00000010:3.1:1775773780.462177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f620af23e00. 00000080:00000010:3.1:1775773790.719174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e896a0. 00000080:00000010:3.1:1775773790.723195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89cc0. 00000080:00000010:3.1:1775773790.723204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd040. 00000080:00000010:3.1:1775773790.723209:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e896c0. 00000080:00000010:3.1:1775773790.723213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e892a0. 00000080:00000010:3.1:1775773790.723215:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89220. 00000080:00000010:3.1:1775773790.723217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e899a0. 00000080:00000010:3.1:1775773790.723221:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e898a0. 00000080:00000010:3.1:1775773790.723226:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8c60. 00000080:00000010:3.1:1775773790.723233:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd780. 00000080:00000010:3.1:1775773790.723236:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd3a0. 00000080:00000010:3.1:1775773790.723241:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb440. 00000080:00000010:3.1:1775773790.723245:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb3c0. 00000080:00000010:3.1:1775773790.723247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89380. 00000080:00000010:3.1:1775773790.723250:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd760. 00000080:00000010:3.1:1775773790.723253:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd5a0. 00000080:00000010:3.1:1775773790.723255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89c00. 00000080:00000010:3.1:1775773790.723258:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e890e0. 00000080:00000010:3.1:1775773790.723261:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89c80. 00000080:00000010:3.1:1775773790.723265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89c20. 00000080:00000010:3.1:1775773790.723267:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdc40. 00000080:00000010:3.1:1775773790.723270:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e47e3c0. 00000080:00000010:3.1:1775773790.723272:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdee0. 00000080:00000010:3.1:1775773790.723275:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb500. 00000080:00000010:3.1:1775773790.723278:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233763fa0. 00000080:00000010:3.1:1775773790.723281:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb2c0. 00000080:00000010:3.1:1775773790.723283:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb9a0. 00000080:00000010:3.1:1775773790.723286:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdd60. 00000080:00000010:3.1:1775773790.723288:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207dccdc0. 00000080:00000010:3.1:1775773790.723291:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8d80. 00000080:00000010:3.1:1775773790.723293:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb4e0. 00000080:00000010:3.1:1775773790.723295:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbfe0. 00000080:00000010:3.1:1775773790.723301:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84dc0. 00000080:00000010:3.1:1775773790.723304:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbea0. 00000080:00000010:3.1:1775773790.723309:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb6c0. 00000080:00000010:3.1:1775773790.723312:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233763740. 00000080:00000010:3.1:1775773790.723314:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdba20. 00000080:00000010:3.1:1775773790.723317:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8460. 00000080:00000010:3.1:1775773790.723319:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb7a0. 00000080:00000010:3.1:1775773790.723322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbb60. 00000080:00000010:3.1:1775773790.723325:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbdc0. 00000080:00000010:3.1:1775773790.723328:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84a00. 00000080:00000010:3.1:1775773790.723333:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdba40. 00000080:00000010:3.1:1775773790.723336:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84de0. 00000080:00000010:3.1:1775773790.723339:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdaa0. 00000080:00000010:3.1:1775773790.723341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbbc0. 00000080:00000010:3.1:1775773790.723344:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdf40. 00000080:00000010:3.1:1775773790.723346:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb880. 00000080:00000010:3.1:1775773790.723348:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd160. 00000080:00000010:3.1:1775773790.723351:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd845c0. 00000080:00000010:3.1:1775773790.723354:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbae0. 00000080:00000010:3.1:1775773790.723357:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84040. 00000080:00000010:3.1:1775773790.723360:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd420. 00000080:00000010:3.1:1775773790.723363:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb520. 00000080:00000010:3.1:1775773790.723366:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84340. 00000080:00000010:3.1:1775773790.723369:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd1a0. 00000080:00000010:3.1:1775773790.723372:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84a60. 00000080:00000010:3.1:1775773790.723375:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84d40. 00000080:00000010:3.1:1775773790.723378:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbd60. 00000080:00000010:3.1:1775773790.723381:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd845a0. 00000080:00000010:3.1:1775773790.723383:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84680. 00000080:00000010:3.1:1775773790.723386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8a40. 00000080:00000010:3.1:1775773790.723389:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84500. 00000080:00000010:3.1:1775773790.723391:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84ae0. 00000080:00000010:3.1:1775773790.723394:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84a80. 00000080:00000010:3.1:1775773790.723397:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdf00. 00000080:00000010:3.1:1775773790.723399:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84e60. 00000080:00000010:3.1:1775773790.723403:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84100. 00000080:00000010:3.1:1775773790.723406:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84f40. 00000080:00000010:3.1:1775773790.723409:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbca0. 00000080:00000010:3.1:1775773790.723411:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84d80. 00000080:00000010:3.1:1775773790.723414:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbb40. 00000080:00000010:3.1:1775773790.723416:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd843c0. 00000080:00000010:3.1:1775773790.723419:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84540. 00000080:00000010:3.1:1775773790.723422:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd0e0. 00000080:00000010:3.1:1775773790.723424:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84260. 00000080:00000010:3.1:1775773790.723429:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84fe0. 00000080:00000010:3.1:1775773790.723432:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84ea0. 00000080:00000010:3.1:1775773790.723435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb3a0. 00000080:00000010:3.1:1775773790.723437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbde0. 00000080:00000010:3.1:1775773790.723440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd680. 00000080:00000010:3.1:1775773790.723442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84420. 00000080:00000010:3.1:1775773790.723445:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84f60. 00000080:00000010:3.1:1775773790.723447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84920. 00000080:00000010:3.1:1775773790.723450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84880. 00000080:00000010:3.1:1775773790.723453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84520. 00000080:00000010:3.1:1775773790.723455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84c00. 00000080:00000010:3.1:1775773790.723458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb560. 00000080:00000010:3.1:1775773790.723461:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb980. 00000080:00000010:3.1:1775773790.723463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbe60. 00000080:00000010:3.1:1775773790.723466:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd060. 00000080:00000010:3.1:1775773790.723469:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8700. 00000080:00000010:3.1:1775773790.723471:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84760. 00000080:00000010:3.1:1775773790.723474:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84240. 00000080:00000010:3.1:1775773790.723476:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84060. 00000080:00000010:3.1:1775773790.723479:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbe20. 00000080:00000010:3.1:1775773790.723482:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd846e0. 00000080:00000010:3.1:1775773790.723485:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84d60. 00000080:00000010:3.1:1775773790.723487:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd848e0. 00000080:00000010:3.1:1775773790.723490:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84160. 00000080:00000010:3.1:1775773790.723492:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd0c0. 00000080:00000010:3.1:1775773790.723495:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd844c0. 00000080:00000010:3.1:1775773790.723497:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84280. 00000080:00000010:3.1:1775773790.723500:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84600. 00000080:00000010:3.1:1775773790.723502:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84120. 00000080:00000010:3.1:1775773790.723505:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb620. 00000080:00000010:3.1:1775773790.723507:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84460. 00000080:00000010:3.1:1775773790.723509:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84b60. 00000080:00000010:3.1:1775773790.723512:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd841a0. 00000080:00000010:3.1:1775773790.723514:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84320. 00000080:00000010:3.1:1775773790.723517:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84b00. 00000080:00000010:3.1:1775773790.723519:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84400. 00000080:00000010:3.1:1775773790.723523:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84380. 00000080:00000010:3.1:1775773790.723526:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84640. 00000080:00000010:3.1:1775773790.723528:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8e80. 00000080:00000010:3.1:1775773790.723531:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84440. 00000080:00000010:3.1:1775773790.723533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd847e0. 00000080:00000010:3.1:1775773790.723536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84900. 00000080:00000010:3.1:1775773790.723538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84360. 00000080:00000010:3.1:1775773790.723540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84000. 00000080:00000010:3.1:1775773790.723543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdd80. 00000080:00000010:3.1:1775773790.723545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd1c0. 00000080:00000010:3.1:1775773790.723548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd843e0. 00000080:00000010:3.1:1775773790.723550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8580. 00000080:00000010:3.1:1775773790.723552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8920. 00000080:00000010:3.1:1775773790.723555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd843a0. 00000080:00000010:3.1:1775773790.723557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84740. 00000080:00000010:3.1:1775773790.723560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd840a0. 00000080:00000010:3.1:1775773790.723562:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a85c0. 00000080:00000010:3.1:1775773790.723564:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd848a0. 00000080:00000010:3.1:1775773790.723567:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd660. 00000080:00000010:3.1:1775773790.723570:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84b20. 00000080:00000010:3.1:1775773790.723573:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84480. 00000080:00000010:3.1:1775773790.723575:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fde60. 00000080:00000010:3.1:1775773790.723578:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd240. 00000080:00000010:3.1:1775773790.723580:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84aa0. 00000080:00000010:3.1:1775773790.723582:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb340. 00000080:00000010:3.1:1775773790.723584:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb900. 00000080:00000010:3.1:1775773790.723587:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb7c0. 00000080:00000010:3.1:1775773790.723589:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84e40. 00000080:00000010:3.1:1775773790.723591:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd100. 00000080:00000010:3.1:1775773790.723594:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd000. 00000080:00000010:3.1:1775773790.723596:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd1e0. 00000080:00000010:3.1:1775773790.723598:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84f20. 00000080:00000010:3.1:1775773790.723611:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd840e0. 00000080:00000010:3.1:1775773790.723613:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84c80. 00000080:00000010:3.1:1775773790.723615:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8a00. 00000080:00000010:3.1:1775773790.723618:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd400. 00000080:00000010:3.1:1775773790.723620:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdde0. 00000080:00000010:3.1:1775773790.723622:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84200. 00000080:00000010:3.1:1775773790.723625:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd847a0. 00000080:00000010:3.1:1775773790.723627:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd845e0. 00000080:00000010:3.1:1775773790.723629:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb760. 00000080:00000010:3.1:1775773790.723631:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84820. 00000080:00000010:3.1:1775773790.723633:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84f00. 00000080:00000010:3.1:1775773790.723635:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84a20. 00000080:00000010:3.1:1775773790.723638:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdae0. 00000080:00000010:3.1:1775773790.723640:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd841e0. 00000080:00000010:3.1:1775773790.723642:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb2a0. 00000080:00000010:3.1:1775773790.723644:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd860. 00000080:00000010:3.1:1775773790.723646:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdb60. 00000080:00000010:3.1:1775773790.723648:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd8a0. 00000080:00000010:3.1:1775773790.723650:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84940. 00000080:00000010:3.1:1775773790.723652:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd720. 00000080:00000010:3.1:1775773790.723654:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb740. 00000080:00000010:3.1:1775773790.723656:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84ce0. 00000080:00000010:3.1:1775773790.723658:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84ba0. 00000080:00000010:3.1:1775773790.723660:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd849a0. 00000080:00000010:3.1:1775773790.723661:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84d20. 00000080:00000010:3.1:1775773790.723663:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dd84660. 00000080:00000010:3.1:1775773790.723675:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6200025d80. 00010000:00000010:3.1:1775773790.723678:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bae700. 00010000:00000010:3.1:1775773790.723681:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cffa00. 00010000:00000010:3.1:1775773790.723684:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233baea00. 00010000:00000010:3.1:1775773790.723686:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cff000. 00010000:00000010:3.1:1775773790.723687:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd50c00. 00010000:00000010:3.1:1775773790.723689:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226054e00. 00010000:00000010:3.1:1775773790.723691:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bae100. 00010000:00000010:3.1:1775773790.723693:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cffe00. 00010000:00000010:3.1:1775773790.723694:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bae600. 00010000:00000010:3.1:1775773790.723696:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cff200. 00010000:00000010:3.1:1775773790.723698:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233baef00. 00010000:00000010:3.1:1775773790.723700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cfe400. 00010000:00000010:3.1:1775773790.723701:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bae000. 00010000:00000010:3.1:1775773790.723703:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cfe800. 00010000:00000010:3.1:1775773790.723717:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bae500. 00010000:00000010:3.1:1775773790.723720:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623491ce00. 00010000:00000010:3.1:1775773790.723722:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233baeb00. 00010000:00000010:3.1:1775773790.723724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623491c800. 00010000:00000010:3.1:1775773790.723725:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233baec00. 00010000:00000010:3.1:1775773790.723727:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226054400. 00010000:00000010:3.1:1775773790.723729:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bae300. 00010000:00000010:3.1:1775773790.723730:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226054600. 00010000:00000010:3.1:1775773790.723732:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265dd500. 00010000:00000010:3.1:1775773790.723735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c604200. 00010000:00000010:3.1:1775773790.723736:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265ddb00. 00010000:00000010:3.1:1775773790.723738:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c604c00. 00010000:00000010:3.1:1775773790.723739:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bae200. 00010000:00000010:3.1:1775773790.723741:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623491d800. 00010000:00000010:3.1:1775773790.723743:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd50300. 00010000:00000010:3.1:1775773790.723745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc90e00. 00010000:00000010:3.1:1775773790.723747:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd50200. 00010000:00000010:3.1:1775773790.723749:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc91600. 00010000:00000010:3.1:1775773790.723751:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210df8b00. 00010000:00000010:3.1:1775773790.723753:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623491c600. 00010000:00000010:3.1:1775773790.723755:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd50d00. 00010000:00000010:3.1:1775773790.723757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623491c000. 00010000:00000010:3.1:1775773790.723758:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233baed00. 00010000:00000010:3.1:1775773790.723760:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623491ca00. 00010000:00000010:3.1:1775773790.723762:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bae400. 00010000:00000010:3.1:1775773790.723764:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623491d400. 00010000:00000010:3.1:1775773790.723766:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d87800. 00010000:00000010:3.1:1775773790.723768:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc91a00. 00010000:00000010:3.1:1775773790.723769:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bae800. 00010000:00000010:3.1:1775773790.723771:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623491dc00. 00010000:00000010:3.1:1775773790.723773:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d87e00. 00010000:00000010:3.1:1775773790.723775:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc90600. 00010000:00000010:3.1:1775773790.723777:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265ddc00. 00010000:00000010:3.1:1775773790.723779:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c605600. 00010000:00000010:3.1:1775773790.723781:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233baee00. 00010000:00000010:3.1:1775773790.723782:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623491cc00. 00010000:00000010:3.1:1775773790.723784:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265dd300. 00010000:00000010:3.1:1775773790.723785:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c605000. 00010000:00000010:3.1:1775773790.723787:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265dd200. 00010000:00000010:3.1:1775773790.723789:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c604800. 00010000:00000010:3.1:1775773790.723790:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d87900. 00010000:00000010:3.1:1775773790.723792:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc90a00. 00010000:00000010:3.1:1775773790.723794:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265f8500. 00010000:00000010:3.1:1775773790.723816:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623491d600. 00010000:00000010:3.1:1775773790.723818:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210df8c00. 00010000:00000010:3.1:1775773790.723819:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225805600. 00010000:00000010:3.1:1775773790.723821:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265ddd00. 00010000:00000010:3.1:1775773790.723823:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c605400. 00010000:00000010:3.1:1775773790.723825:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265dd400. 00010000:00000010:3.1:1775773790.723826:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c605c00. 00010000:00000010:3.1:1775773790.723828:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265dd800. 00010000:00000010:3.1:1775773790.723829:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c605a00. 00010000:00000010:3.1:1775773790.723831:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265f8b00. 00010000:00000010:3.1:1775773790.723833:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623491c200. 00010000:00000010:3.1:1775773790.723835:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265dd900. 00010000:00000010:3.1:1775773790.723837:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c605800. 00010000:00000010:3.1:1775773790.723838:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265dd700. 00010000:00000010:3.1:1775773790.723840:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c604000. 00010000:00000010:3.1:1775773790.723842:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210df8300. 00010000:00000010:3.1:1775773790.723844:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623491da00. 00010000:00000010:3.1:1775773790.723845:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265f8c00. 00010000:00000010:3.1:1775773790.723847:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c604e00. 00010000:00000010:3.1:1775773790.723850:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233766100. 00010000:00000010:3.1:1775773790.723852:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225804e00. 00010000:00000010:3.1:1775773790.723854:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265dda00. 00010000:00000010:3.1:1775773790.723856:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c604a00. 00010000:00000010:3.1:1775773790.723857:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265dd100. 00010000:00000010:3.1:1775773790.723859:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c605200. 00010000:00000010:3.1:1775773790.723860:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265f8300. 00010000:00000010:3.1:1775773790.723862:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c605e00. 00010000:00000010:3.1:1775773790.723864:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265f8200. 00010000:00000010:3.1:1775773790.723865:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623491d000. 00010000:00000010:3.1:1775773790.723867:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206633c00. 00010000:00000010:3.1:1775773790.723869:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c604400. 00010000:00000010:3.1:1775773790.723871:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265f8d00. 00010000:00000010:3.1:1775773790.723872:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623491de00. 00010000:00000010:3.1:1775773790.723874:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d87700. 00010000:00000010:3.1:1775773790.723876:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc90800. 00010000:00000010:3.1:1775773790.723877:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206633300. 00010000:00000010:3.1:1775773790.723879:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c604600. 00010000:00000010:3.1:1775773790.723881:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265f8400. 00010000:00000010:3.1:1775773790.723882:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc91400. 00010000:00000010:3.1:1775773790.723884:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206633200. 00010000:00000010:3.1:1775773790.723886:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bff4400. 00010000:00000010:3.1:1775773790.723908:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265f8800. 00010000:00000010:3.1:1775773790.723910:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc91200. 00010000:00000010:3.1:1775773790.723912:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265f8e00. 00010000:00000010:3.1:1775773790.723914:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623491d200. 00010000:00000010:3.1:1775773790.723916:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d87a00. 00010000:00000010:3.1:1775773790.723918:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bff5800. 00010000:00000010:3.1:1775773790.723920:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233766600. 00010000:00000010:3.1:1775773790.723921:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623491c400. 00010000:00000010:3.1:1775773790.723926:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206633d00. 00010000:00000010:3.1:1775773790.723928:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc90400. 00010000:00000010:3.1:1775773790.723930:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206633400. 00010000:00000010:3.1:1775773790.723932:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bff5e00. 00010000:00000010:3.1:1775773790.723935:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265f8900. 00010000:00000010:3.1:1775773790.723937:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e405c00. 00010000:00000010:3.1:1775773790.723939:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265f8700. 00010000:00000010:3.1:1775773790.723941:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc90c00. 00010000:00000010:3.1:1775773790.723942:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265f8a00. 00010000:00000010:3.1:1775773790.723944:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e404c00. 00010000:00000010:3.1:1775773790.723946:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265f8100. 00010000:00000010:3.1:1775773790.723947:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e405600. 00010000:00000010:3.1:1775773790.723949:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d87100. 00010000:00000010:3.1:1775773790.723951:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bff4200. 00010000:00000010:3.1:1775773790.723952:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d87600. 00010000:00000010:3.1:1775773790.723954:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e404e00. 00010000:00000010:3.1:1775773790.723955:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265f8600. 00010000:00000010:3.1:1775773790.723957:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e404200. 00010000:00000010:3.1:1775773790.723959:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233766f00. 00010000:00000010:3.1:1775773790.723961:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225804200. 00010000:00000010:3.1:1775773790.723962:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265f8f00. 00010000:00000010:3.1:1775773790.723964:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e405a00. 00010000:00000010:3.1:1775773790.723965:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d87f00. 00010000:00000010:3.1:1775773790.723967:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e405000. 00010000:00000010:3.1:1775773790.723968:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206633800. 00010000:00000010:3.1:1775773790.723970:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e405e00. 00010000:00000010:3.1:1775773790.723972:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d87000. 00010000:00000010:3.1:1775773790.723973:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc91c00. 00010000:00000010:3.1:1775773790.723975:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233766000. 00010000:00000010:3.1:1775773790.723977:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e405200. 00010000:00000010:3.1:1775773790.723978:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265f8000. 00010000:00000010:3.1:1775773790.723980:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e404400. 00010000:00000010:3.1:1775773790.723982:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623397c800. 00010000:00000010:3.1:1775773790.723985:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e404800. 00010000:00000010:3.1:1775773790.723986:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206633e00. 00010000:00000010:3.1:1775773790.723988:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bff4800. 00010000:00000010:3.1:1775773790.723990:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623397ce00. 00010000:00000010:3.1:1775773790.723992:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e405800. 00010000:00000010:3.1:1775773790.723994:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623397c900. 00010000:00000010:3.1:1775773790.723995:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bff5600. 00010000:00000010:3.1:1775773790.723997:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233766500. 00010000:00000010:3.1:1775773790.723998:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e404600. 00010000:00000010:3.1:1775773790.724000:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623397c700. 00010000:00000010:3.1:1775773790.724002:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e404000. 00010000:00000010:3.1:1775773790.724003:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233766b00. 00010000:00000010:3.1:1775773790.724005:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc91000. 00010000:00000010:3.1:1775773790.724006:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623397ca00. 00010000:00000010:3.1:1775773790.724008:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e404a00. 00010000:00000010:3.1:1775773790.724010:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623397c100. 00010000:00000010:3.1:1775773790.724011:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e405400. 00010000:00000010:3.1:1775773790.724013:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233766c00. 00010000:00000010:3.1:1775773790.724015:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6204cb7c00. 00010000:00000010:3.1:1775773790.724017:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206633900. 00010000:00000010:3.1:1775773790.724018:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bff4a00. 00010000:00000010:3.1:1775773790.724020:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206633700. 00010000:00000010:3.1:1775773790.724021:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bff5400. 00010000:00000010:3.1:1775773790.724023:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233766300. 00010000:00000010:3.1:1775773790.724025:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc90200. 00010000:00000010:3.1:1775773790.724026:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623397c600. 00010000:00000010:3.1:1775773790.724028:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6204cb6c00. 00010000:00000010:3.1:1775773790.724030:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623397cf00. 00010000:00000010:3.1:1775773790.724031:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6204cb7600. 00010000:00000010:3.1:1775773790.724033:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623397c000. 00010000:00000010:3.1:1775773790.724034:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6204cb6e00. 00010000:00000010:3.1:1775773790.724036:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623397c500. 00010000:00000010:3.1:1775773790.724038:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6204cb6200. 00010000:00000010:3.1:1775773790.724040:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623397cb00. 00010000:00000010:3.1:1775773790.724042:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6204cb7a00. 00010000:00000010:3.1:1775773790.724043:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623397cc00. 00010000:00000010:3.1:1775773790.724045:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6204cb7000. 00010000:00000010:3.1:1775773790.724047:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623397c300. 00010000:00000010:3.1:1775773790.724048:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bff4600. 00010000:00000010:3.1:1775773790.724050:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233766200. 00010000:00000010:3.1:1775773790.724051:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bff5c00. 00010000:00000010:3.1:1775773790.724053:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233766d00. 00010000:00000010:3.1:1775773790.724055:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bff4c00. 00010000:00000010:3.1:1775773790.724056:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206633a00. 00010000:00000010:3.1:1775773790.724058:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc91800. 00010000:00000010:3.1:1775773790.724059:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233766400. 00010000:00000010:3.1:1775773790.724061:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225805a00. 00010000:00000010:3.1:1775773790.724062:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623397c200. 00010000:00000010:3.1:1775773790.724064:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6204cb7e00. 00010000:00000010:3.1:1775773790.724065:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623397cd00. 00010000:00000010:3.1:1775773790.724067:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6204cb7200. 00010000:00000010:3.1:1775773790.724068:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623397c400. 00010000:00000010:3.1:1775773790.724070:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6204cb6400. 00010000:00000010:3.1:1775773790.724072:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab7f700. 00010000:00000010:3.1:1775773790.724074:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bff4000. 00010000:00000010:3.1:1775773790.724075:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab7fa00. 00010000:00000010:3.1:1775773790.724077:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6204cb6800. 00010000:00000010:3.1:1775773790.724078:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab7f100. 00010000:00000010:3.1:1775773790.724080:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6204cb7800. 00010000:00000010:3.1:1775773790.724082:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab7f600. 00010000:00000010:3.1:1775773790.724083:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6204cb6600. 00010000:00000010:3.1:1775773790.724085:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab7ff00. 00010000:00000010:3.1:1775773790.724086:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6204cb6000. 00010000:00000010:3.1:1775773790.724088:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d87500. 00010000:00000010:3.1:1775773790.724090:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc91e00. 00010000:00000010:3.1:1775773790.724092:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab7f000. 00010000:00000010:3.1:1775773790.724093:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6204cb6a00. 00010000:00000010:3.1:1775773790.724095:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab7f500. 00010000:00000010:3.1:1775773790.724097:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6204cb7400. 00010000:00000010:3.1:1775773790.724099:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d87b00. 00010000:00000010:3.1:1775773790.724101:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8dd200. 00010000:00000010:3.1:1775773790.724102:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab7fb00. 00010000:00000010:3.1:1775773790.724104:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8dc400. 00010000:00000010:3.1:1775773790.724106:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206633100. 00010000:00000010:3.1:1775773790.724107:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bff5000. 00010000:00000010:3.1:1775773790.724109:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab7fc00. 00010000:00000010:3.1:1775773790.724111:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8dc800. 00010000:00000010:3.1:1775773790.724112:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab7f300. 00010000:00000010:3.1:1775773790.724114:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8dd800. 00010000:00000010:3.1:1775773790.724115:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab7f200. 00010000:00000010:3.1:1775773790.724117:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8dc600. 00010000:00000010:3.1:1775773790.724118:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab7fd00. 00010000:00000010:3.1:1775773790.724120:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8dc000. 00010000:00000010:3.1:1775773790.724121:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab7f400. 00010000:00000010:3.1:1775773790.724123:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8dca00. 00010000:00000010:3.1:1775773790.724124:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab7f800. 00010000:00000010:3.1:1775773790.724126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8dd400. 00010000:00000010:3.1:1775773790.724128:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab7fe00. 00010000:00000010:3.1:1775773790.724129:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8ddc00. 00010000:00000010:3.1:1775773790.724131:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620ab7f900. 00010000:00000010:3.1:1775773790.724145:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8dcc00. 00010000:00000010:3.1:1775773790.724147:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233766800. 00010000:00000010:3.1:1775773790.724149:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225805000. 00010000:00000010:3.1:1775773790.724150:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233766e00. 00010000:00000010:3.1:1775773790.724152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8dd600. 00010000:00000010:3.1:1775773790.724154:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e4e6400. 00010000:00000010:3.1:1775773790.724157:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8dce00. 00010000:00000010:3.1:1775773790.724159:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206633600. 00010000:00000010:3.1:1775773790.724160:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8dc200. 00010000:00000010:3.1:1775773790.724162:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e4e6800. 00010000:00000010:3.1:1775773790.724164:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8dda00. 00010000:00000010:3.1:1775773790.724166:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e4e6e00. 00010000:00000010:3.1:1775773790.724168:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8dd000. 00010000:00000010:3.1:1775773790.724169:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e4e6900. 00010000:00000010:3.1:1775773790.724171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc90000. 00010000:00000010:3.1:1775773790.724176:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e4e6700. 00010000:00000010:3.1:1775773790.724178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226471400. 00010000:00000010:3.1:1775773790.724180:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e4e6a00. 00010000:00000010:3.1:1775773790.724182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8dde00. 00010000:00000010:3.1:1775773790.724183:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233766900. 00010000:00000010:3.1:1775773790.724185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225805e00. 00010000:00000010:3.1:1775773790.724187:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233766700. 00010000:00000010:3.1:1775773790.724188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225804400. 00010000:00000010:3.1:1775773790.724190:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d87c00. 00010000:00000010:3.1:1775773790.724192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de92600. 00010000:00000010:3.1:1775773790.724194:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233766a00. 00010000:00000010:3.1:1775773790.724196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de93800. 00010000:00000010:3.1:1775773790.724198:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e4e6100. 00010000:00000010:3.1:1775773790.724200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de93a00. 00010000:00000010:3.1:1775773790.724201:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e122a00. 00010000:00000010:3.1:1775773790.724203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225805200. 00010000:00000010:3.1:1775773790.724205:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e4e6600. 00010000:00000010:3.1:1775773790.724206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de93000. 00010000:00000010:3.1:1775773790.724208:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e4e6f00. 00010000:00000010:3.1:1775773790.724210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226471000. 00010000:00000010:3.1:1775773790.724212:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e4e6000. 00010000:00000010:3.1:1775773790.724213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de93e00. 00010000:00000010:3.1:1775773790.724215:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e122100. 00010000:00000010:3.1:1775773790.724217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de93200. 00010000:00000010:3.1:1775773790.724219:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206633f00. 00010000:00000010:3.1:1775773790.724221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226470800. 00010000:00000010:3.1:1775773790.724223:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e4e6500. 00010000:00000010:3.1:1775773790.724226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226470600. 00010000:00000010:3.1:1775773790.724229:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e4e6b00. 00010000:00000010:3.1:1775773790.724232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de92400. 00010000:00000010:3.1:1775773790.724235:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d87300. 00010000:00000010:3.1:1775773790.724238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bff4e00. 00010000:00000010:3.1:1775773790.724240:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d87200. 00010000:00000010:3.1:1775773790.724243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bff5a00. 00010000:00000010:3.1:1775773790.724245:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206633000. 00010000:00000010:3.1:1775773790.724248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622bff5200. 00010000:00000010:3.1:1775773790.724251:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e4e6c00. 00010000:00000010:3.1:1775773790.724253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de92800. 00010000:00000010:3.1:1775773790.724256:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d87d00. 00010000:00000010:3.1:1775773790.724258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226471200. 00010000:00000010:3.1:1775773790.724261:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d87400. 00010000:00000010:3.1:1775773790.724268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226470c00. 00010000:00000010:3.1:1775773790.724272:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62057d4a00. 00010000:00000010:3.1:1775773790.724275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226470a00. 00010000:00000010:3.1:1775773790.724278:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e122600. 00010000:00000010:3.1:1775773790.724281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de92000. 00010000:00000010:3.1:1775773790.724284:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e122f00. 00010000:00000010:3.1:1775773790.724287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de92a00. 00010000:00000010:3.1:1775773790.724289:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e4e6300. 00010000:00000010:3.1:1775773790.724292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de93400. 00010000:00000010:3.1:1775773790.724295:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e122000. 00010000:00000010:3.1:1775773790.724298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225804800. 00010000:00000010:3.1:1775773790.724300:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62057d4100. 00010000:00000010:3.1:1775773790.724303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226471a00. 00010000:00000010:3.1:1775773790.724306:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206633500. 00010000:00000010:3.1:1775773790.724310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226471e00. 00010000:00000010:3.1:1775773790.724312:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e4e6200. 00010000:00000010:3.1:1775773790.724315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de93c00. 00010000:00000010:3.1:1775773790.724318:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206633b00. 00010000:00000010:3.1:1775773790.724320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de92c00. 00010000:00000010:3.1:1775773790.724322:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e4e6d00. 00010000:00000010:3.1:1775773790.724324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de93600. 00010000:00000010:3.1:1775773790.724326:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225977e00. 00010000:00000010:3.1:1775773790.724328:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623364b600. 00010000:00000010:3.1:1775773790.724330:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233754500. 00010000:00000010:3.1:1775773790.724332:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de92e00. 00010000:00000010:3.1:1775773790.724333:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233754b00. 00010000:00000010:3.1:1775773790.724335:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de92200. 00010000:00000010:3.1:1775773790.724336:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e122500. 00010000:00000010:3.1:1775773790.724338:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e102e00. 00010000:00000010:3.1:1775773790.724340:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233754c00. 00010000:00000010:3.1:1775773790.724341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226470000. 00010000:00000010:3.1:1775773790.724343:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233754300. 00010000:00000010:3.1:1775773790.724345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e102200. 00010000:00000010:3.1:1775773790.724346:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6225977c00. 00010000:00000010:3.1:1775773790.724348:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623364b800. 00010000:00000010:3.1:1775773790.724350:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62057d4600. 00010000:00000010:3.1:1775773790.724352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226470200. 00010000:00000010:3.1:1775773790.724353:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62057d4f00. 00010000:00000010:3.1:1775773790.724355:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226471600. 00010000:00000010:3.1:1775773790.724356:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62057d4000. 00010000:00000010:3.1:1775773790.724358:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226470e00. 00010000:00000010:3.1:1775773790.724359:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233754200. 00010000:00000010:3.1:1775773790.724361:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e103a00. 00010000:00000010:3.1:1775773790.724363:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62057d4500. 00010000:00000010:3.1:1775773790.724368:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226470400. 00010000:00000010:3.1:1775773790.724373:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233754d00. 00010000:00000010:3.1:1775773790.724375:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623364b400. 00010000:00000010:3.1:1775773790.724379:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233754400. 00010000:00000010:3.1:1775773790.724382:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e103000. 00010000:00000010:3.1:1775773790.724383:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233754800. 00010000:00000010:3.1:1775773790.724385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e103e00. 00010000:00000010:3.1:1775773790.724387:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233754e00. 00010000:00000010:3.1:1775773790.724388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e103200. 00010000:00000010:3.1:1775773790.724390:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233754900. 00010000:00000010:3.1:1775773790.724393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e102400. 00010000:00000010:3.1:1775773790.724395:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e122b00. 00010000:00000010:3.1:1775773790.724399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e102800. 00000080:00000010:3.1:1775773800.940180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb400. 00000080:00000010:3.1:1775773800.940191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbac0. 00000080:00000010:3.1:1775773800.940195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15b00. 00000080:00000010:3.1:1775773800.940203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89a40. 00000080:00000010:3.1:1775773800.944168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89b20. 00000080:00000010:3.1:1775773800.944177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89600. 00000080:00000010:3.1:1775773800.944181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15240. 00000080:00000010:3.1:1775773800.944187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbf60. 00000080:00000010:3.1:1775773800.944191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e899c0. 00000080:00000010:3.1:1775773800.944194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb8e0. 00000080:00000010:3.1:1775773800.944197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15740. 00000080:00000010:3.1:1775773800.944202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a152a0. 00000080:00000010:3.1:1775773800.944205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89fa0. 00000080:00000010:3.1:1775773800.944208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e891a0. 00000080:00000010:3.1:1775773800.944213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89320. 00000080:00000010:3.1:1775773800.944218:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89840. 00000080:00000010:3.1:1775773800.944221:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89b40. 00000080:00000010:3.1:1775773800.944225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89ee0. 00000080:00000010:3.1:1775773800.944230:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89d80. 00000080:00000010:3.1:1775773800.944235:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e891e0. 00000080:00000010:3.1:1775773800.944238:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89b00. 00000080:00000010:3.1:1775773800.944241:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e896e0. 00000080:00000010:3.1:1775773800.944244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb460. 00000080:00000010:3.1:1775773800.944248:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89e00. 00000080:00000010:3.1:1775773800.944251:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15160. 00000080:00000010:3.1:1775773800.944254:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89560. 00000080:00000010:3.1:1775773800.944257:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb480. 00000080:00000010:3.1:1775773800.944259:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb820. 00000080:00000010:3.1:1775773800.944262:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89760. 00000080:00000010:3.1:1775773800.944265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89440. 00000080:00000010:3.1:1775773800.944269:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89940. 00000080:00000010:3.1:1775773800.944272:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb320. 00000080:00000010:3.1:1775773800.944275:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb720. 00000080:00000010:3.1:1775773800.944277:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89640. 00000080:00000010:3.1:1775773800.944280:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e897c0. 00000080:00000010:3.1:1775773800.944283:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89720. 00000080:00000010:3.1:1775773800.944286:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89f40. 00000080:00000010:3.1:1775773800.944289:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89200. 00000080:00000010:3.1:1775773800.944292:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89260. 00000080:00000010:3.1:1775773800.944295:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb840. 00000080:00000010:3.1:1775773800.944298:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89100. 00000080:00000010:3.1:1775773800.944303:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89500. 00000080:00000010:3.1:1775773800.944307:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89660. 00000080:00000010:3.1:1775773800.944311:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb9c0. 00000080:00000010:3.1:1775773800.944315:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89bc0. 00000080:00000010:3.1:1775773800.944320:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8c00. 00000080:00000010:3.1:1775773800.944326:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e897e0. 00000080:00000010:3.1:1775773800.944330:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620849b380. 00000080:00000010:3.1:1775773800.944335:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620849b7a0. 00000080:00000010:3.1:1775773800.944339:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdba60. 00000080:00000010:3.1:1775773800.944346:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e895c0. 00000080:00000010:3.1:1775773800.944353:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbc20. 00000080:00000010:3.1:1775773800.944361:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8e40. 00000080:00000010:3.1:1775773800.944365:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89e20. 00000080:00000010:3.1:1775773800.944368:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620a8ba780. 00000080:00000010:3.1:1775773800.944371:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6206ad05a0. 00000080:00000010:3.1:1775773800.944375:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbce0. 00000080:00000010:3.1:1775773800.944378:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89340. 00000080:00000010:3.1:1775773800.944381:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6200025fc0. 00000080:00000010:3.1:1775773800.944386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb180. 00000080:00000010:3.1:1775773800.944388:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8960. 00000080:00000010:3.1:1775773800.944392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd0a0. 00000080:00000010:3.1:1775773800.944395:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89fe0. 00000080:00000010:3.1:1775773800.944398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89800. 00000080:00000010:3.1:1775773800.944400:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89040. 00000080:00000010:3.1:1775773800.944403:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89f60. 00000080:00000010:3.1:1775773800.944406:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb280. 00000080:00000010:3.1:1775773800.944408:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb540. 00000080:00000010:3.1:1775773800.944411:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb380. 00000080:00000010:3.1:1775773800.944413:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a87e0. 00000080:00000010:3.1:1775773800.944416:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89400. 00000080:00000010:3.1:1775773800.944420:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89540. 00000080:00000010:3.1:1775773800.944423:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89f00. 00000080:00000010:3.1:1775773800.944425:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89000. 00000080:00000010:3.1:1775773800.944428:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbaa0. 00000080:00000010:3.1:1775773800.944430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb420. 00000080:00000010:3.1:1775773800.944433:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89360. 00000080:00000010:3.1:1775773800.944435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb0e0. 00000080:00000010:3.1:1775773800.944438:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e895a0. 00000080:00000010:3.1:1775773800.944440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6200025a80. 00000080:00000010:3.1:1775773800.944443:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbc60. 00000080:00000010:3.1:1775773800.944445:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb220. 00000080:00000010:3.1:1775773800.944448:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb780. 00000080:00000010:3.1:1775773800.944450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fda60. 00000080:00000010:3.1:1775773800.944452:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb5a0. 00000080:00000010:3.1:1775773800.944455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89160. 00000080:00000010:3.1:1775773800.944458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89ce0. 00000080:00000010:3.1:1775773800.944460:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89060. 00000080:00000010:3.1:1775773800.944463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb660. 00000080:00000010:3.1:1775773800.944465:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89180. 00000080:00000010:3.1:1775773800.944467:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89080. 00000080:00000010:3.1:1775773800.944469:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89680. 00000080:00000010:3.1:1775773800.944472:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89480. 00000080:00000010:3.1:1775773800.944474:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89a80. 00000080:00000010:3.1:1775773800.944476:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbd20. 00000080:00000010:3.1:1775773800.944480:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd8c0. 00000080:00000010:3.1:1775773800.944483:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e892e0. 00000080:00000010:3.1:1775773800.944485:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89de0. 00000080:00000010:3.1:1775773800.944488:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd9a0. 00000080:00000010:3.1:1775773800.944490:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e895e0. 00000080:00000010:3.1:1775773800.944493:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89140. 00000080:00000010:3.1:1775773800.944496:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb120. 00000080:00000010:3.1:1775773800.944499:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb1c0. 00000080:00000010:3.1:1775773800.944501:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb6e0. 00000080:00000010:3.1:1775773800.944504:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdda0. 00000080:00000010:3.1:1775773800.944506:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb860. 00000080:00000010:3.1:1775773800.944508:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb580. 00000080:00000010:3.1:1775773800.944511:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdd20. 00000080:00000010:3.1:1775773800.944513:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdd40. 00000080:00000010:3.1:1775773800.944515:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89b60. 00000080:00000010:3.1:1775773800.944518:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89e40. 00000080:00000010:3.1:1775773800.944520:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89da0. 00000080:00000010:3.1:1775773800.944523:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e894a0. 00000080:00000010:3.1:1775773800.944525:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb000. 00000080:00000010:3.1:1775773800.944541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89900. 00000080:00000010:3.1:1775773800.944544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89620. 00000080:00000010:3.1:1775773800.944546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e893e0. 00000080:00000010:3.1:1775773800.944548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd640. 00000080:00000010:3.1:1775773800.944551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89520. 00000080:00000010:3.1:1775773800.944554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89420. 00000080:00000010:3.1:1775773800.944556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbc40. 00000080:00000010:3.1:1775773800.944559:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd5e0. 00000080:00000010:3.1:1775773800.944562:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8f60. 00000080:00000010:3.1:1775773800.944564:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89f80. 00000080:00000010:3.1:1775773800.944566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8c20. 00000080:00000010:3.1:1775773800.944569:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89e60. 00000080:00000010:3.1:1775773800.944571:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89ea0. 00000080:00000010:3.1:1775773800.944573:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8040. 00000080:00000010:3.1:1775773800.944577:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a83a0. 00000080:00000010:3.1:1775773800.944580:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbd80. 00000080:00000010:3.1:1775773800.944582:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd2c0. 00000080:00000010:3.1:1775773800.944585:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e899e0. 00000080:00000010:3.1:1775773800.944587:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd200. 00000080:00000010:3.1:1775773800.944590:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbd40. 00000080:00000010:3.1:1775773800.944592:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8680. 00000080:00000010:3.1:1775773800.944594:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89d60. 00000080:00000010:3.1:1775773800.944597:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89ca0. 00000080:00000010:3.1:1775773800.944599:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8ce0. 00000080:00000010:3.1:1775773800.944601:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e894c0. 00000080:00000010:3.1:1775773800.944603:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8f40. 00000080:00000010:3.1:1775773800.944606:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd940. 00000080:00000010:3.1:1775773800.944608:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e890a0. 00000080:00000010:3.1:1775773800.944610:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb0c0. 00000080:00000010:3.1:1775773800.944612:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd7c0. 00000080:00000010:3.1:1775773800.944615:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbbe0. 00000080:00000010:3.1:1775773800.944617:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89ac0. 00000080:00000010:3.1:1775773800.944620:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e891c0. 00000080:00000010:3.1:1775773800.944622:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbe40. 00000080:00000010:3.1:1775773800.944624:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89b80. 00000080:00000010:3.1:1775773800.944626:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89880. 00000080:00000010:3.1:1775773800.944628:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89f20. 00000080:00000010:3.1:1775773800.944630:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89c40. 00000080:00000010:3.1:1775773800.944633:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8840. 00000080:00000010:3.1:1775773800.944635:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e897a0. 00000080:00000010:3.1:1775773800.944637:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89d00. 00000080:00000010:3.1:1775773800.944639:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89a00. 00000080:00000010:3.1:1775773800.944641:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89c60. 00000080:00000010:3.1:1775773800.944643:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8ec0. 00000080:00000010:3.1:1775773800.944645:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb260. 00000080:00000010:3.1:1775773800.944647:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8ea0. 00000080:00000010:3.1:1775773800.944649:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fde20. 00010000:00000010:3.1:1775773800.944653:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62262abc00. 00010000:00000010:3.1:1775773800.944656:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202fc1a00. 00010000:00000010:3.1:1775773800.944660:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62337be000. 00010000:00000010:3.1:1775773800.944662:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202fc0e00. 00010000:00000010:3.1:1775773800.944664:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62262ab300. 00010000:00000010:3.1:1775773800.944667:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62082cee00. 00010000:00000010:3.1:1775773800.944672:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e155f00. 00010000:00000010:3.1:1775773800.944674:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335d8e00. 00010000:00000010:3.1:1775773800.944676:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e155000. 00010000:00000010:3.1:1775773800.944678:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335d8200. 00010000:00000010:3.1:1775773800.944680:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e155500. 00010000:00000010:3.1:1775773800.944682:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335d9a00. 00010000:00000010:3.1:1775773800.944684:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62337be500. 00010000:00000010:3.1:1775773800.944686:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62082ce200. 00010000:00000010:3.1:1775773800.944688:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62337beb00. 00010000:00000010:3.1:1775773800.944689:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202fc0800. 00010000:00000010:3.1:1775773800.944691:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e155b00. 00010000:00000010:3.1:1775773800.944693:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335d9000. 00010000:00000010:3.1:1775773800.944695:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62337bec00. 00010000:00000010:3.1:1775773800.944696:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202fc1200. 00010000:00000010:3.1:1775773800.944698:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e155c00. 00010000:00000010:3.1:1775773800.944700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62082cf600. 00010000:00000010:3.1:1775773800.944703:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205608300. 00010000:00000010:3.1:1775773800.944705:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62082ce600. 00010000:00000010:3.1:1775773800.944707:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c46fe00. 00010000:00000010:3.1:1775773800.944709:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335d9e00. 00010000:00000010:3.1:1775773800.944711:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e155300. 00010000:00000010:3.1:1775773800.944712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335d9200. 00010000:00000010:3.1:1775773800.944714:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e155200. 00010000:00000010:3.1:1775773800.944716:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335d8400. 00010000:00000010:3.1:1775773800.944717:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205608200. 00010000:00000010:3.1:1775773800.944719:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335d8800. 00010000:00000010:3.1:1775773800.944721:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e155d00. 00010000:00000010:3.1:1775773800.944723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d6800. 00010000:00000010:3.1:1775773800.944725:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e155400. 00010000:00000010:3.1:1775773800.944728:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d7800. 00010000:00000010:3.1:1775773800.944729:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e155800. 00010000:00000010:3.1:1775773800.944731:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d6600. 00010000:00000010:3.1:1775773800.944733:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e155e00. 00010000:00000010:3.1:1775773800.944735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d6000. 00010000:00000010:3.1:1775773800.944737:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c46f900. 00010000:00000010:3.1:1775773800.944739:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d6a00. 00010000:00000010:3.1:1775773800.944740:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e155900. 00010000:00000010:3.1:1775773800.944742:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d7400. 00010000:00000010:3.1:1775773800.944743:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e155700. 00010000:00000010:3.1:1775773800.944745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202fc0600. 00010000:00000010:3.1:1775773800.944747:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e155a00. 00010000:00000010:3.1:1775773800.944748:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d7c00. 00010000:00000010:3.1:1775773800.944750:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e155100. 00010000:00000010:3.1:1775773800.944751:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62082cfa00. 00010000:00000010:3.1:1775773800.944753:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e155600. 00010000:00000010:3.1:1775773800.944755:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d6c00. 00010000:00000010:3.1:1775773800.944756:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205608d00. 00010000:00000010:3.1:1775773800.944758:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202fc0400. 00010000:00000010:3.1:1775773800.944761:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfaf000. 00010000:00000010:3.1:1775773800.944763:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210299a00. 00010000:00000010:3.1:1775773800.944765:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfaf500. 00010000:00000010:3.1:1775773800.944767:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d7600. 00010000:00000010:3.1:1775773800.944769:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfafb00. 00010000:00000010:3.1:1775773800.944771:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d6e00. 00010000:00000010:3.1:1775773800.944772:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfafc00. 00010000:00000010:3.1:1775773800.944774:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d6200. 00010000:00000010:3.1:1775773800.944776:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6209e34600. 00010000:00000010:3.1:1775773800.944779:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210298200. 00010000:00000010:3.1:1775773800.944780:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205608400. 00010000:00000010:3.1:1775773800.944782:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210298000. 00010000:00000010:3.1:1775773800.944784:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfaf300. 00010000:00000010:3.1:1775773800.944785:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d7a00. 00010000:00000010:3.1:1775773800.944787:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfaf200. 00010000:00000010:3.1:1775773800.944789:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d7000. 00010000:00000010:3.1:1775773800.944791:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfafd00. 00010000:00000010:3.1:1775773800.944793:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d7e00. 00010000:00000010:3.1:1775773800.944795:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfaf400. 00010000:00000010:3.1:1775773800.944796:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d7200. 00010000:00000010:3.1:1775773800.944798:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfaf800. 00010000:00000010:3.1:1775773800.944800:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d6400. 00010000:00000010:3.1:1775773800.944801:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfafe00. 00010000:00000010:3.1:1775773800.944803:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206200e00. 00010000:00000010:3.1:1775773800.944805:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfaf900. 00010000:00000010:3.1:1775773800.944807:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210299600. 00010000:00000010:3.1:1775773800.944809:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6209e34f00. 00010000:00000010:3.1:1775773800.944811:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206200200. 00010000:00000010:3.1:1775773800.944812:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfaf700. 00010000:00000010:3.1:1775773800.944815:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206201a00. 00010000:00000010:3.1:1775773800.944816:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfafa00. 00010000:00000010:3.1:1775773800.944818:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206201000. 00010000:00000010:3.1:1775773800.944819:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205608800. 00010000:00000010:3.1:1775773800.944821:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210299c00. 00010000:00000010:3.1:1775773800.944823:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfaf100. 00010000:00000010:3.1:1775773800.944825:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206201e00. 00010000:00000010:3.1:1775773800.944826:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c46f700. 00010000:00000010:3.1:1775773800.944828:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca47200. 00010000:00000010:3.1:1775773800.944830:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfaf600. 00010000:00000010:3.1:1775773800.944831:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206201200. 00010000:00000010:3.1:1775773800.944833:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6209e34000. 00010000:00000010:3.1:1775773800.944835:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62082cf000. 00010000:00000010:3.1:1775773800.944836:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6209e34500. 00010000:00000010:3.1:1775773800.944838:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62082cfe00. 00010000:00000010:3.1:1775773800.944840:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c46fa00. 00010000:00000010:3.1:1775773800.944841:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210298c00. 00010000:00000010:3.1:1775773800.944843:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cfaff00. 00010000:00000010:3.1:1775773800.944845:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206200400. 00010000:00000010:3.1:1775773800.944847:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205608e00. 00010000:00000010:3.1:1775773800.944849:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210299400. 00010000:00000010:3.1:1775773800.944853:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206188100. 00010000:00000010:3.1:1775773800.944855:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206200800. 00010000:00000010:3.1:1775773800.944857:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206188600. 00010000:00000010:3.1:1775773800.944858:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206201800. 00010000:00000010:3.1:1775773800.944860:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c46f100. 00010000:00000010:3.1:1775773800.944862:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62082cec00. 00010000:00000010:3.1:1775773800.944863:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206188f00. 00010000:00000010:3.1:1775773800.944865:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62082cf400. 00010000:00000010:3.1:1775773800.944867:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206188000. 00010000:00000010:3.1:1775773800.944869:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62082cea00. 00010000:00000010:3.1:1775773800.944870:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206188500. 00010000:00000010:3.1:1775773800.944872:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206200600. 00010000:00000010:3.1:1775773800.944874:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd50400. 00010000:00000010:3.1:1775773800.944876:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62082cfc00. 00010000:00000010:3.1:1775773800.944878:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206188b00. 00010000:00000010:3.1:1775773800.944879:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210298400. 00010000:00000010:3.1:1775773800.944881:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c46f600. 00010000:00000010:3.1:1775773800.944883:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca46400. 00010000:00000010:3.1:1775773800.944885:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206188c00. 00010000:00000010:3.1:1775773800.944887:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62082ce800. 00010000:00000010:3.1:1775773800.944888:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206188300. 00010000:00000010:3.1:1775773800.944890:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206200000. 00010000:00000010:3.1:1775773800.944892:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206188200. 00010000:00000010:3.1:1775773800.944893:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206200a00. 00010000:00000010:3.1:1775773800.944895:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c46ff00. 00010000:00000010:3.1:1775773800.944896:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206201400. 00010000:00000010:3.1:1775773800.944898:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c46f000. 00010000:00000010:3.1:1775773800.944900:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca46800. 00010000:00000010:3.1:1775773800.944901:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205608900. 00010000:00000010:3.1:1775773800.944904:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210298800. 00010000:00000010:3.1:1775773800.944905:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206188d00. 00010000:00000010:3.1:1775773800.944907:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210298600. 00010000:00000010:3.1:1775773800.944909:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205608700. 00010000:00000010:3.1:1775773800.944911:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210298a00. 00010000:00000010:3.1:1775773800.944913:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c46f500. 00010000:00000010:3.1:1775773800.944914:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca47800. 00010000:00000010:3.1:1775773800.944916:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206188400. 00010000:00000010:3.1:1775773800.944918:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206201c00. 00010000:00000010:3.1:1775773800.944919:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205608a00. 00010000:00000010:3.1:1775773800.944921:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206200c00. 00010000:00000010:3.1:1775773800.944922:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206188800. 00010000:00000010:3.1:1775773800.944924:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206201600. 00010000:00000010:3.1:1775773800.944925:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206188e00. 00010000:00000010:3.1:1775773800.944928:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d9600. 00010000:00000010:3.1:1775773800.944930:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205608100. 00010000:00000010:3.1:1775773800.944931:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210299e00. 00010000:00000010:3.1:1775773800.944933:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205608600. 00010000:00000010:3.1:1775773800.944935:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210298e00. 00010000:00000010:3.1:1775773800.944936:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206188900. 00010000:00000010:3.1:1775773800.944938:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d8e00. 00010000:00000010:3.1:1775773800.944940:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205608f00. 00010000:00000010:3.1:1775773800.944942:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210299800. 00010000:00000010:3.1:1775773800.944944:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210df8200. 00010000:00000010:3.1:1775773800.944946:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d8200. 00010000:00000010:3.1:1775773800.944948:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206188700. 00010000:00000010:3.1:1775773800.944950:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62082cf800. 00010000:00000010:3.1:1775773800.944951:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd50800. 00010000:00000010:3.1:1775773800.944953:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210299000. 00010000:00000010:3.1:1775773800.944955:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd50e00. 00010000:00000010:3.1:1775773800.944956:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210299200. 00010000:00000010:3.1:1775773800.944958:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206188a00. 00010000:00000010:3.1:1775773800.944961:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dda1600. 00010000:00000010:3.1:1775773800.944963:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd50900. 00010000:00000010:3.1:1775773800.944965:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62082cf200. 00010000:00000010:3.1:1775773800.944967:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622be95d00. 00010000:00000010:3.1:1775773800.944996:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dda1200. 00010000:00000010:3.1:1775773800.944999:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622be95400. 00010000:00000010:3.1:1775773800.945001:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d9a00. 00010000:00000010:3.1:1775773800.945003:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210df8d00. 00010000:00000010:3.1:1775773800.945005:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d9000. 00010000:00000010:3.1:1775773800.945007:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622be95800. 00010000:00000010:3.1:1775773800.945008:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d9e00. 00010000:00000010:3.1:1775773800.945010:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210df8400. 00010000:00000010:3.1:1775773800.945011:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dda1800. 00010000:00000010:3.1:1775773800.945013:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd50700. 00010000:00000010:3.1:1775773800.945015:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d9200. 00010000:00000010:3.1:1775773800.945017:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622be95e00. 00010000:00000010:3.1:1775773800.945018:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d8400. 00010000:00000010:3.1:1775773800.945020:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622be95900. 00010000:00000010:3.1:1775773800.945022:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d8800. 00010000:00000010:3.1:1775773800.945023:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622be95700. 00010000:00000010:3.1:1775773800.945025:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d9800. 00010000:00000010:3.1:1775773800.945026:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622be95a00. 00010000:00000010:3.1:1775773800.945028:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d8600. 00010000:00000010:3.1:1775773800.945030:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622be95100. 00010000:00000010:3.1:1775773800.945031:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dda1e00. 00010000:00000010:3.1:1775773800.945033:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622be95600. 00010000:00000010:3.1:1775773800.945035:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226054200. 00010000:00000010:3.1:1775773800.945036:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622be95f00. 00010000:00000010:3.1:1775773800.945038:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d8000. 00010000:00000010:3.1:1775773800.945039:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622be95000. 00010000:00000010:3.1:1775773800.945041:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d8a00. 00010000:00000010:3.1:1775773800.945042:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622be95500. 00010000:00000010:3.1:1775773800.945044:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226054000. 00010000:00000010:3.1:1775773800.945046:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd50a00. 00010000:00000010:3.1:1775773800.945048:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d9400. 00010000:00000010:3.1:1775773800.945050:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622be95b00. 00010000:00000010:3.1:1775773800.945052:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d9c00. 00010000:00000010:3.1:1775773800.945053:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210df8800. 00010000:00000010:3.1:1775773800.945056:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226055a00. 00010000:00000010:3.1:1775773800.945057:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205608000. 00010000:00000010:3.1:1775773800.945059:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dda0000. 00010000:00000010:3.1:1775773800.945061:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205608500. 00010000:00000010:3.1:1775773800.945063:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dda1400. 00010000:00000010:3.1:1775773800.945065:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210df8e00. 00010000:00000010:3.1:1775773800.945066:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226054800. 00010000:00000010:3.1:1775773800.945068:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd50100. 00010000:00000010:3.1:1775773800.945070:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dda0e00. 00010000:00000010:3.1:1775773800.945071:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205608b00. 00010000:00000010:3.1:1775773800.945073:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dda0600. 00010000:00000010:3.1:1775773800.945075:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd50600. 00010000:00000010:3.1:1775773800.945076:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226055c00. 00010000:00000010:3.1:1775773800.945078:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205608c00. 00010000:00000010:3.1:1775773800.945082:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226055e00. 00010000:00000010:3.1:1775773800.945084:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622be95c00. 00010000:00000010:3.1:1775773800.945086:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d8c00. 00010000:00000010:3.1:1775773800.945087:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622be95300. 00010000:00000010:3.1:1775773800.945089:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc35400. 00010000:00000010:3.1:1775773800.945115:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622be95200. 00010000:00000010:3.1:1775773800.945117:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc35200. 00010000:00000010:3.1:1775773800.945120:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c045d00. 00010000:00000010:3.1:1775773800.945122:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc34400. 00010000:00000010:3.1:1775773800.945124:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265dde00. 00010000:00000010:3.1:1775773800.945126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dda0400. 00010000:00000010:3.1:1775773800.945128:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c045400. 00010000:00000010:3.1:1775773800.945130:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc34800. 00010000:00000010:3.1:1775773800.945132:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c045800. 00010000:00000010:3.1:1775773800.945147:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc35800. 00010000:00000010:3.1:1775773800.945150:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c045e00. 00010000:00000010:3.1:1775773800.945152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc34600. 00010000:00000010:3.1:1775773800.945154:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c045900. 00010000:00000010:3.1:1775773800.945156:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226055400. 00010000:00000010:3.1:1775773800.945158:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c045700. 00010000:00000010:3.1:1775773800.945160:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc34000. 00010000:00000010:3.1:1775773800.945161:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c045a00. 00010000:00000010:3.1:1775773800.945163:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc34a00. 00010000:00000010:3.1:1775773800.945164:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c045100. 00010000:00000010:3.1:1775773800.945166:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dda0800. 00010000:00000010:3.1:1775773800.945168:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd50f00. 00010000:00000010:3.1:1775773800.945170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226054a00. 00010000:00000010:3.1:1775773800.945171:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c045600. 00010000:00000010:3.1:1775773800.945173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca46600. 00010000:00000010:3.1:1775773800.945175:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c045f00. 00010000:00000010:3.1:1775773800.945177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc35c00. 00010000:00000010:3.1:1775773800.945178:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210df8900. 00010000:00000010:3.1:1775773800.945180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca46000. 00010000:00000010:3.1:1775773800.945182:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c045000. 00010000:00000010:3.1:1775773800.945183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc34c00. 00010000:00000010:3.1:1775773800.945185:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210df8700. 00010000:00000010:3.1:1775773800.945187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc35600. 00010000:00000010:3.1:1775773800.945188:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210df8a00. 00010000:00000010:3.1:1775773800.945190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca46a00. 00010000:00000010:3.1:1775773800.945191:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c045500. 00010000:00000010:3.1:1775773800.945193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca47400. 00010000:00000010:3.1:1775773800.945195:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd50000. 00010000:00000010:3.1:1775773800.945196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dda1000. 00010000:00000010:3.1:1775773800.945198:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c045b00. 00010000:00000010:3.1:1775773800.945201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226055800. 00010000:00000010:3.1:1775773800.945202:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c045c00. 00010000:00000010:3.1:1775773800.945204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc34e00. 00010000:00000010:3.1:1775773800.945206:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c045300. 00010000:00000010:3.1:1775773800.945208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226054c00. 00010000:00000010:3.1:1775773800.945210:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265dd600. 00010000:00000010:3.1:1775773800.945211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dda1c00. 00010000:00000010:3.1:1775773800.945213:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210df8100. 00010000:00000010:3.1:1775773800.945214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca47c00. 00010000:00000010:3.1:1775773800.945216:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c045200. 00010000:00000010:3.1:1775773800.945218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc34200. 00010000:00000010:3.1:1775773800.945221:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cf35b00. 00010000:00000010:3.1:1775773800.945222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc35a00. 00010000:00000010:3.1:1775773800.945224:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210df8600. 00010000:00000010:3.1:1775773800.945226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca46c00. 00010000:00000010:3.1:1775773800.945227:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210df8f00. 00010000:00000010:3.1:1775773800.945229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc35000. 00010000:00000010:3.1:1775773800.945231:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cf35c00. 00010000:00000010:3.1:1775773800.945232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca47600. 00010000:00000010:3.1:1775773800.945234:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265ddf00. 00010000:00000010:3.1:1775773800.945236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226055600. 00010000:00000010:3.1:1775773800.945237:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cf35300. 00010000:00000010:3.1:1775773800.945239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc35e00. 00010000:00000010:3.1:1775773800.945241:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cf35200. 00010000:00000010:3.1:1775773800.945242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dda0a00. 00010000:00000010:3.1:1775773800.945244:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd50500. 00010000:00000010:3.1:1775773800.945246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226055000. 00010000:00000010:3.1:1775773800.945248:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cf35d00. 00010000:00000010:3.1:1775773800.945250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dda0c00. 00010000:00000010:3.1:1775773800.945252:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cf35400. 00010000:00000010:3.1:1775773800.945254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cff800. 00010000:00000010:3.1:1775773800.945255:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cf35800. 00010000:00000010:3.1:1775773800.945258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cfe600. 00010000:00000010:3.1:1775773800.945260:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cf35e00. 00010000:00000010:3.1:1775773800.945262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dda0200. 00010000:00000010:3.1:1775773800.945263:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cf35900. 00010000:00000010:3.1:1775773800.945265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cfe000. 00010000:00000010:3.1:1775773800.945267:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cf35700. 00010000:00000010:3.1:1775773800.945269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cfea00. 00010000:00000010:3.1:1775773800.945271:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cf35a00. 00010000:00000010:3.1:1775773800.945272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cff400. 00010000:00000010:3.1:1775773800.945274:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cf35100. 00010000:00000010:3.1:1775773800.945275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cffc00. 00010000:00000010:3.1:1775773800.945277:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210df8000. 00010000:00000010:3.1:1775773800.945279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca46e00. 00010000:00000010:3.1:1775773800.945280:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cf35600. 00010000:00000010:3.1:1775773800.945282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cfec00. 00010000:00000010:3.1:1775773800.945284:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cf35f00. 00010000:00000010:3.1:1775773800.945285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cff600. 00010000:00000010:3.1:1775773800.945287:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cf35000. 00010000:00000010:3.1:1775773800.945288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cfee00. 00010000:00000010:3.1:1775773800.945290:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cf35500. 00010000:00000010:3.1:1775773800.945291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cfe200. 00010000:00000010:3.1:1775773800.945297:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210df8500. 00010000:00000010:3.1:1775773800.945303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca46200. 00010000:00000010:3.1:1775773800.945306:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62265dd000. 00010000:00000010:3.1:1775773800.945312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dda1a00. 00010000:00000010:3.1:1775773800.945318:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cd50b00. 00010000:00000010:3.1:1775773800.945329:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225804c00. 00010000:00000010:3.1:1775773800.945337:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233bae900. 00010000:00000010:3.1:1775773800.945343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226055200. 00000080:00000010:3.1:1775773811.175164:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15320. 00000080:00000010:3.1:1775773811.179174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572ff20. 00000080:00000010:3.1:1775773811.179183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f220. 00000080:00000010:3.1:1775773811.179187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572fb20. 00000080:00000010:3.1:1775773811.179191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c252e0. 00000080:00000010:3.1:1775773811.179195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15920. 00000080:00000010:3.1:1775773811.179198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25680. 00000080:00000010:3.1:1775773811.179201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f140. 00000080:00000010:3.1:1775773811.179203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f020. 00000080:00000010:3.1:1775773811.179206:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572fc60. 00000080:00000010:3.1:1775773811.179216:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15680. 00000080:00000010:3.1:1775773811.179219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f5e0. 00000080:00000010:3.1:1775773811.179222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572fc40. 00000080:00000010:3.1:1775773811.179224:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f080. 00000080:00000010:3.1:1775773811.179227:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f5c0. 00000080:00000010:3.1:1775773811.179232:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572fb60. 00000080:00000010:3.1:1775773811.179234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f360. 00000080:00000010:3.1:1775773811.179237:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f380. 00000080:00000010:3.1:1775773811.179240:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb3e0. 00000080:00000010:3.1:1775773811.179243:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb1a0. 00000080:00000010:3.1:1775773811.179246:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f940. 00000080:00000010:3.1:1775773811.179251:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbfc0. 00000080:00000010:3.1:1775773811.179254:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572fd60. 00000080:00000010:3.1:1775773811.179259:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8340. 00000080:00000010:3.1:1775773811.179263:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb6a0. 00000080:00000010:3.1:1775773811.179266:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15880. 00000080:00000010:3.1:1775773811.179273:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15b60. 00000080:00000010:3.1:1775773811.179276:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbda0. 00000080:00000010:3.1:1775773811.179278:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f700. 00000080:00000010:3.1:1775773811.179281:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f760. 00000080:00000010:3.1:1775773811.179284:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f500. 00000080:00000010:3.1:1775773811.179287:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8440. 00000080:00000010:3.1:1775773811.179290:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f5a0. 00000080:00000010:3.1:1775773811.179292:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15660. 00000080:00000010:3.1:1775773811.179295:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8160. 00000080:00000010:3.1:1775773811.179297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f9a0. 00000080:00000010:3.1:1775773811.179300:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8660. 00000080:00000010:3.1:1775773811.179302:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a89e0. 00000080:00000010:3.1:1775773811.179305:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572fb00. 00000080:00000010:3.1:1775773811.179307:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8400. 00000080:00000010:3.1:1775773811.179310:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f3c0. 00000080:00000010:3.1:1775773811.179312:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f800. 00000080:00000010:3.1:1775773811.179315:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572ff80. 00000080:00000010:3.1:1775773811.179317:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f9c0. 00000080:00000010:3.1:1775773811.179319:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f160. 00000080:00000010:3.1:1775773811.179322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572fb40. 00000080:00000010:3.1:1775773811.179324:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15280. 00000080:00000010:3.1:1775773811.179327:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb8c0. 00000080:00000010:3.1:1775773811.179329:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572fce0. 00000080:00000010:3.1:1775773811.179332:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f4c0. 00000080:00000010:3.1:1775773811.179336:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f7a0. 00000080:00000010:3.1:1775773811.179338:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f920. 00000080:00000010:3.1:1775773811.179341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15340. 00000080:00000010:3.1:1775773811.179343:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15d00. 00000080:00000010:3.1:1775773811.179346:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572fe80. 00000080:00000010:3.1:1775773811.179385:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f180. 00000080:00000010:3.1:1775773811.179388:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f460. 00000080:00000010:3.1:1775773811.179391:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f600. 00000080:00000010:3.1:1775773811.179393:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f200. 00000080:00000010:3.1:1775773811.179396:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f1a0. 00000080:00000010:3.1:1775773811.179398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572fa20. 00000080:00000010:3.1:1775773811.179400:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f260. 00000080:00000010:3.1:1775773811.179402:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f320. 00000080:00000010:3.1:1775773811.179404:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572fc80. 00000080:00000010:3.1:1775773811.179406:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f780. 00000080:00000010:3.1:1775773811.179409:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbb00. 00000080:00000010:3.1:1775773811.179411:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a84e0. 00000080:00000010:3.1:1775773811.179414:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15780. 00000080:00000010:3.1:1775773811.179417:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb240. 00000080:00000010:3.1:1775773811.179419:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15c80. 00000080:00000010:3.1:1775773811.179422:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f720. 00000080:00000010:3.1:1775773811.179424:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbe80. 00000080:00000010:3.1:1775773811.179428:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a159e0. 00000080:00000010:3.1:1775773811.179430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbcc0. 00000080:00000010:3.1:1775773811.179433:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb5e0. 00000080:00000010:3.1:1775773811.179436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f2e0. 00000080:00000010:3.1:1775773811.179439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f680. 00000080:00000010:3.1:1775773811.179442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572fe20. 00000080:00000010:3.1:1775773811.179444:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15540. 00000080:00000010:3.1:1775773811.179447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233763e80. 00000080:00000010:3.1:1775773811.179450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6206ad09e0. 00000080:00000010:3.1:1775773811.179453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620849baa0. 00000080:00000010:3.1:1775773811.179456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620849b300. 00000080:00000010:3.1:1775773811.179458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233edbce0. 00000080:00000010:3.1:1775773811.179460:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb700. 00000080:00000010:3.1:1775773811.179463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15ea0. 00000080:00000010:3.1:1775773811.179467:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e893c0. 00000080:00000010:3.1:1775773811.179469:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15c00. 00000080:00000010:3.1:1775773811.179472:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6233e7b320. 00000080:00000010:3.1:1775773811.179475:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620849b8a0. 00000080:00000010:3.1:1775773811.179477:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15440. 00000080:00000010:3.1:1775773811.179479:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89920. 00000080:00000010:3.1:1775773811.179482:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89ba0. 00000080:00000010:3.1:1775773811.179485:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbc00. 00000080:00000010:3.1:1775773811.179487:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdba00. 00000080:00000010:3.1:1775773811.179490:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8fc0. 00000080:00000010:3.1:1775773811.179492:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8780. 00000080:00000010:3.1:1775773811.179494:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620849b5c0. 00000080:00000010:3.1:1775773811.179497:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620849b500. 00000080:00000010:3.1:1775773811.179499:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89ec0. 00000080:00000010:3.1:1775773811.179502:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15080. 00000080:00000010:3.1:1775773811.179504:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89820. 00000080:00000010:3.1:1775773811.179507:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e898c0. 00000080:00000010:3.1:1775773811.179509:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89a60. 00000080:00000010:3.1:1775773811.179511:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89d20. 00000080:00000010:3.1:1775773811.179513:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbee0. 00000080:00000010:3.1:1775773811.179516:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89aa0. 00000080:00000010:3.1:1775773811.179518:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbe00. 00000080:00000010:3.1:1775773811.179520:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15fa0. 00000080:00000010:3.1:1775773811.179522:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15940. 00000080:00000010:3.1:1775773811.179525:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbf20. 00000080:00000010:3.1:1775773811.179528:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb7e0. 00000080:00000010:3.1:1775773811.179530:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8a80. 00000080:00000010:3.1:1775773811.179532:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb5c0. 00000080:00000010:3.1:1775773811.179535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15aa0. 00000080:00000010:3.1:1775773811.179538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15020. 00000080:00000010:3.1:1775773811.179540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8d40. 00000080:00000010:3.1:1775773811.179542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89460. 00000080:00000010:3.1:1775773811.179544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15600. 00000080:00000010:3.1:1775773811.179546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8ac0. 00000080:00000010:3.1:1775773811.179549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89020. 00000080:00000010:3.1:1775773811.179552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb4c0. 00000080:00000010:3.1:1775773811.179554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89740. 00000080:00000010:3.1:1775773811.179556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89dc0. 00000080:00000010:3.1:1775773811.179558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8220. 00000080:00000010:3.1:1775773811.179561:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a153c0. 00000080:00000010:3.1:1775773811.179563:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdbf80. 00000080:00000010:3.1:1775773811.179564:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15200. 00000080:00000010:3.1:1775773811.179566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89960. 00000080:00000010:3.1:1775773811.179569:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8820. 00000080:00000010:3.1:1775773811.179571:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e892c0. 00000080:00000010:3.1:1775773811.179573:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89780. 00000080:00000010:3.1:1775773811.179575:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e898e0. 00000080:00000010:3.1:1775773811.179577:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6202e89fc0. 00000080:00000010:3.1:1775773811.179579:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15000. 00000080:00000010:3.1:1775773811.179581:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb300. 00000080:00000010:3.1:1775773811.179583:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb960. 00000080:00000010:3.1:1775773811.179585:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8da0. 00000080:00000010:3.1:1775773811.179587:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb640. 00010000:00000010:3.1:1775773811.179590:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa5000. 00010000:00000010:3.1:1775773811.179593:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210407800. 00010000:00000010:3.1:1775773811.179596:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207a64f00. 00010000:00000010:3.1:1775773811.179598:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddb6600. 00010000:00000010:3.1:1775773811.179600:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207a64000. 00010000:00000010:3.1:1775773811.179602:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddb6000. 00010000:00000010:3.1:1775773811.179604:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234418100. 00010000:00000010:3.1:1775773811.179606:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dcbcc00. 00010000:00000010:3.1:1775773811.179608:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa5500. 00010000:00000010:3.1:1775773811.179611:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e281400. 00010000:00000010:3.1:1775773811.179613:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62264d7d00. 00010000:00000010:3.1:1775773811.179614:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210407600. 00010000:00000010:3.1:1775773811.179616:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207a64500. 00010000:00000010:3.1:1775773811.179618:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e280000. 00010000:00000010:3.1:1775773811.179620:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234418600. 00010000:00000010:3.1:1775773811.179622:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dcbd600. 00010000:00000010:3.1:1775773811.179624:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234418f00. 00010000:00000010:3.1:1775773811.179625:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dcbce00. 00010000:00000010:3.1:1775773811.179627:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207a64b00. 00010000:00000010:3.1:1775773811.179629:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dcbc200. 00010000:00000010:3.1:1775773811.179630:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234418000. 00010000:00000010:3.1:1775773811.179632:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210407400. 00010000:00000010:3.1:1775773811.179634:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6209e34b00. 00010000:00000010:3.1:1775773811.179636:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dcbda00. 00010000:00000010:3.1:1775773811.179638:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207a64c00. 00010000:00000010:3.1:1775773811.179640:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dcbd000. 00010000:00000010:3.1:1775773811.179641:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa5b00. 00010000:00000010:3.1:1775773811.179643:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dcbde00. 00010000:00000010:3.1:1775773811.179645:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6209e34c00. 00010000:00000010:3.1:1775773811.179646:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dcbd200. 00010000:00000010:3.1:1775773811.179648:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234418500. 00010000:00000010:3.1:1775773811.179649:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dcbc400. 00010000:00000010:3.1:1775773811.179651:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234418b00. 00010000:00000010:3.1:1775773811.179653:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dcbc800. 00010000:00000010:3.1:1775773811.179654:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234418c00. 00010000:00000010:3.1:1775773811.179656:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dcbd800. 00010000:00000010:3.1:1775773811.179657:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6209e34300. 00010000:00000010:3.1:1775773811.179659:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e280600. 00010000:00000010:3.1:1775773811.179660:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234418300. 00010000:00000010:3.1:1775773811.179662:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e280800. 00010000:00000010:3.1:1775773811.179664:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207a64300. 00010000:00000010:3.1:1775773811.179666:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dcbc600. 00010000:00000010:3.1:1775773811.179668:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa5c00. 00010000:00000010:3.1:1775773811.179670:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e280400. 00010000:00000010:3.1:1775773811.179671:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234418200. 00010000:00000010:3.1:1775773811.179673:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dcbc000. 00010000:00000010:3.1:1775773811.179674:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207a64200. 00010000:00000010:3.1:1775773811.179676:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d862200. 00010000:00000010:3.1:1775773811.179678:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207a64d00. 00010000:00000010:3.1:1775773811.179679:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e280c00. 00010000:00000010:3.1:1775773811.179681:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6209e34200. 00010000:00000010:3.1:1775773811.179683:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210406a00. 00010000:00000010:3.1:1775773811.179684:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207a64400. 00010000:00000010:3.1:1775773811.179686:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210406800. 00010000:00000010:3.1:1775773811.179687:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234418d00. 00010000:00000010:3.1:1775773811.179689:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e281c00. 00010000:00000010:3.1:1775773811.179690:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234418400. 00010000:00000010:3.1:1775773811.179692:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dcbca00. 00010000:00000010:3.1:1775773811.179694:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cb10700. 00010000:00000010:3.1:1775773811.179696:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dcbd400. 00010000:00000010:3.1:1775773811.179698:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cb10a00. 00010000:00000010:3.1:1775773811.179699:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dcbdc00. 00010000:00000010:3.1:1775773811.179701:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207a64800. 00010000:00000010:3.1:1775773811.179702:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d863a00. 00010000:00000010:3.1:1775773811.179704:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cb10100. 00010000:00000010:3.1:1775773811.179706:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233736a00. 00010000:00000010:3.1:1775773811.179708:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cb10600. 00010000:00000010:3.1:1775773811.179709:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210407e00. 00010000:00000010:3.1:1775773811.179711:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207a64e00. 00010000:00000010:3.1:1775773811.179712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d863000. 00010000:00000010:3.1:1775773811.179714:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cb10f00. 00010000:00000010:3.1:1775773811.179717:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233737400. 00010000:00000010:3.1:1775773811.179718:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207a64900. 00010000:00000010:3.1:1775773811.179720:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d863e00. 00010000:00000010:3.1:1775773811.179722:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207a64700. 00010000:00000010:3.1:1775773811.179724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d863200. 00010000:00000010:3.1:1775773811.179725:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cb10000. 00010000:00000010:3.1:1775773811.179727:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233737c00. 00010000:00000010:3.1:1775773811.179729:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cb10500. 00010000:00000010:3.1:1775773811.179730:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d862400. 00010000:00000010:3.1:1775773811.179732:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62337be300. 00010000:00000010:3.1:1775773811.179734:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233736c00. 00010000:00000010:3.1:1775773811.179735:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207a64a00. 00010000:00000010:3.1:1775773811.179737:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233737600. 00010000:00000010:3.1:1775773811.179738:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cb10b00. 00010000:00000010:3.1:1775773811.179740:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233736e00. 00010000:00000010:3.1:1775773811.179741:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cb10c00. 00010000:00000010:3.1:1775773811.179743:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233736200. 00010000:00000010:3.1:1775773811.179744:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62337be200. 00010000:00000010:3.1:1775773811.179746:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233737a00. 00010000:00000010:3.1:1775773811.179748:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cb10300. 00010000:00000010:3.1:1775773811.179749:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233737000. 00010000:00000010:3.1:1775773811.179751:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6209e34d00. 00010000:00000010:3.1:1775773811.179752:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210406600. 00010000:00000010:3.1:1775773811.179755:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210ac0000. 00010000:00000010:3.1:1775773811.179757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e281000. 00010000:00000010:3.1:1775773811.179758:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cb10200. 00010000:00000010:3.1:1775773811.179760:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233737e00. 00010000:00000010:3.1:1775773811.179762:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cb10d00. 00010000:00000010:3.1:1775773811.179763:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233737200. 00010000:00000010:3.1:1775773811.179765:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210ac0500. 00010000:00000010:3.1:1775773811.179766:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233736400. 00010000:00000010:3.1:1775773811.179768:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cb10400. 00010000:00000010:3.1:1775773811.179770:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233736800. 00010000:00000010:3.1:1775773811.179772:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6209e34400. 00010000:00000010:3.1:1775773811.179773:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210406e00. 00010000:00000010:3.1:1775773811.179775:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210ac0b00. 00010000:00000010:3.1:1775773811.179778:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262a9c00. 00010000:00000010:3.1:1775773811.179780:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62337bed00. 00010000:00000010:3.1:1775773811.179781:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233737800. 00010000:00000010:3.1:1775773811.179783:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cb10800. 00010000:00000010:3.1:1775773811.179784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233736600. 00010000:00000010:3.1:1775773811.179786:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cb10e00. 00010000:00000010:3.1:1775773811.179787:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233736000. 00010000:00000010:3.1:1775773811.179789:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cb10900. 00010000:00000010:3.1:1775773811.179791:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b59e00. 00010000:00000010:3.1:1775773811.179793:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210ac0c00. 00010000:00000010:3.1:1775773811.179794:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b59200. 00010000:00000010:3.1:1775773811.179797:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202c0b00. 00010000:00000010:3.1:1775773811.179799:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b58400. 00010000:00000010:3.1:1775773811.179801:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202c0c00. 00010000:00000010:3.1:1775773811.179803:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b58800. 00010000:00000010:3.1:1775773811.179804:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202c0300. 00010000:00000010:3.1:1775773811.179806:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b59800. 00010000:00000010:3.1:1775773811.179807:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202c0200. 00010000:00000010:3.1:1775773811.179809:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b58600. 00010000:00000010:3.1:1775773811.179811:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202c0d00. 00010000:00000010:3.1:1775773811.179812:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b58000. 00010000:00000010:3.1:1775773811.179814:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202c0400. 00010000:00000010:3.1:1775773811.179816:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b58a00. 00010000:00000010:3.1:1775773811.179817:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210ac0300. 00010000:00000010:3.1:1775773811.179819:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e280200. 00010000:00000010:3.1:1775773811.179821:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210ac0200. 00010000:00000010:3.1:1775773811.179823:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d862800. 00010000:00000010:3.1:1775773811.179824:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210ac0d00. 00010000:00000010:3.1:1775773811.179827:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262a8800. 00010000:00000010:3.1:1775773811.179828:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210ac0400. 00010000:00000010:3.1:1775773811.179830:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e281800. 00010000:00000010:3.1:1775773811.179832:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6209e34800. 00010000:00000010:3.1:1775773811.179834:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262a8e00. 00010000:00000010:3.1:1775773811.179836:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202c0800. 00010000:00000010:3.1:1775773811.179837:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b59400. 00010000:00000010:3.1:1775773811.179839:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202c0e00. 00010000:00000010:3.1:1775773811.179840:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e281e00. 00010000:00000010:3.1:1775773811.179842:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6209e34e00. 00010000:00000010:3.1:1775773811.179844:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262a9200. 00010000:00000010:3.1:1775773811.179845:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210ac0800. 00010000:00000010:3.1:1775773811.179847:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e281200. 00010000:00000010:3.1:1775773811.179848:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202c0900. 00010000:00000010:3.1:1775773811.179850:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e280e00. 00010000:00000010:3.1:1775773811.179852:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202c0700. 00010000:00000010:3.1:1775773811.179853:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b59c00. 00010000:00000010:3.1:1775773811.179855:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202c0a00. 00010000:00000010:3.1:1775773811.179856:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b58c00. 00010000:00000010:3.1:1775773811.179858:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202c0100. 00010000:00000010:3.1:1775773811.179859:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b59600. 00010000:00000010:3.1:1775773811.179861:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6209e34900. 00010000:00000010:3.1:1775773811.179863:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262a8a00. 00010000:00000010:3.1:1775773811.179864:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202c0600. 00010000:00000010:3.1:1775773811.179866:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b58e00. 00010000:00000010:3.1:1775773811.179867:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202c0f00. 00010000:00000010:3.1:1775773811.179869:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b58200. 00010000:00000010:3.1:1775773811.179870:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202c0000. 00010000:00000010:3.1:1775773811.179872:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b59a00. 00010000:00000010:3.1:1775773811.179873:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62202c0500. 00010000:00000010:3.1:1775773811.179875:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b59000. 00010000:00000010:3.1:1775773811.179877:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205607000. 00010000:00000010:3.1:1775773811.179880:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c3c6c00. 00010000:00000010:3.1:1775773811.179882:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205607500. 00010000:00000010:3.1:1775773811.179884:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e281a00. 00010000:00000010:3.1:1775773811.179885:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62337be400. 00010000:00000010:3.1:1775773811.179887:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262a9600. 00010000:00000010:3.1:1775773811.179889:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205607b00. 00010000:00000010:3.1:1775773811.179891:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c3c7600. 00010000:00000010:3.1:1775773811.179893:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62337be800. 00010000:00000010:3.1:1775773811.179895:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262a9400. 00010000:00000010:3.1:1775773811.179896:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205607c00. 00010000:00000010:3.1:1775773811.179898:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c3c6e00. 00010000:00000010:3.1:1775773811.179899:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205607300. 00010000:00000010:3.1:1775773811.179901:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c3c6200. 00010000:00000010:3.1:1775773811.179903:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62337bee00. 00010000:00000010:3.1:1775773811.179904:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262a8c00. 00010000:00000010:3.1:1775773811.179906:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205607200. 00010000:00000010:3.1:1775773811.179907:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c3c7a00. 00010000:00000010:3.1:1775773811.179909:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62337be900. 00010000:00000010:3.1:1775773811.179910:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c3c7000. 00010000:00000010:3.1:1775773811.179912:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62337be700. 00010000:00000010:3.1:1775773811.179914:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202fc0000. 00010000:00000010:3.1:1775773811.179916:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205607d00. 00010000:00000010:3.1:1775773811.179917:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202fc1600. 00010000:00000010:3.1:1775773811.179919:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210ac0e00. 00010000:00000010:3.1:1775773811.179921:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d863800. 00010000:00000010:3.1:1775773811.179922:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210ac0900. 00010000:00000010:3.1:1775773811.179924:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d862600. 00010000:00000010:3.1:1775773811.179925:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205607400. 00010000:00000010:3.1:1775773811.179927:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c3c7e00. 00010000:00000010:3.1:1775773811.179929:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205607800. 00010000:00000010:3.1:1775773811.179930:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c3c7200. 00010000:00000010:3.1:1775773811.179932:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205607e00. 00010000:00000010:3.1:1775773811.179934:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c3c6400. 00010000:00000010:3.1:1775773811.179936:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62337bea00. 00010000:00000010:3.1:1775773811.179938:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262a8600. 00010000:00000010:3.1:1775773811.179939:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205607900. 00010000:00000010:3.1:1775773811.179941:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c3c6800. 00010000:00000010:3.1:1775773811.179943:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205607700. 00010000:00000010:3.1:1775773811.179944:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c3c7800. 00010000:00000010:3.1:1775773811.179946:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62337be100. 00010000:00000010:3.1:1775773811.179947:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c3c6600. 00010000:00000010:3.1:1775773811.179949:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205607a00. 00010000:00000010:3.1:1775773811.179950:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c3c6000. 00010000:00000010:3.1:1775773811.179952:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210ac0700. 00010000:00000010:3.1:1775773811.179953:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202fc1e00. 00010000:00000010:3.1:1775773811.179955:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205607100. 00010000:00000010:3.1:1775773811.179956:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c3c6a00. 00010000:00000010:3.1:1775773811.179958:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62337be600. 00010000:00000010:3.1:1775773811.179960:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202fc1800. 00010000:00000010:3.1:1775773811.179961:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210ac0a00. 00010000:00000010:3.1:1775773811.179963:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262a8200. 00010000:00000010:3.1:1775773811.179964:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205607600. 00010000:00000010:3.1:1775773811.179966:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262a9a00. 00010000:00000010:3.1:1775773811.179968:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210ac0100. 00010000:00000010:3.1:1775773811.179969:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202fc0a00. 00010000:00000010:3.1:1775773811.179971:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205607f00. 00010000:00000010:3.1:1775773811.179973:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202fc1c00. 00010000:00000010:3.1:1775773811.179974:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210ac0600. 00010000:00000010:3.1:1775773811.179976:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d862000. 00010000:00000010:3.1:1775773811.179977:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62262ab100. 00010000:00000010:3.1:1775773811.179979:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c3c7400. 00010000:00000010:3.1:1775773811.179981:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62262ab200. 00010000:00000010:3.1:1775773811.179982:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262a8400. 00010000:00000010:3.1:1775773811.179984:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6209e34700. 00010000:00000010:3.1:1775773811.179986:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262a9800. 00010000:00000010:3.1:1775773811.179988:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6210ac0f00. 00010000:00000010:3.1:1775773811.179990:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d862a00. 00010000:00000010:3.1:1775773811.179992:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62262abd00. 00010000:00000010:3.1:1775773811.179994:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c3c7c00. 00010000:00000010:3.1:1775773811.179995:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62337bef00. 00010000:00000010:3.1:1775773811.180007:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262a9000. 00010000:00000010:3.1:1775773811.180010:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c46fb00. 00010000:00000010:3.1:1775773811.180011:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d863400. 00010000:00000010:3.1:1775773811.180013:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62262ab400. 00010000:00000010:3.1:1775773811.180015:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62082ce000. 00010000:00000010:3.1:1775773811.180017:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c46fc00. 00010000:00000010:3.1:1775773811.180019:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202fc0c00. 00010000:00000010:3.1:1775773811.180020:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62262ab800. 00010000:00000010:3.1:1775773811.180022:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335d9800. 00010000:00000010:3.1:1775773811.180025:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c46f300. 00010000:00000010:3.1:1775773811.180027:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335d8600. 00010000:00000010:3.1:1775773811.180030:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c46f200. 00010000:00000010:3.1:1775773811.180033:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca47a00. 00010000:00000010:3.1:1775773811.180036:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6209e34a00. 00010000:00000010:3.1:1775773811.180039:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262a8000. 00010000:00000010:3.1:1775773811.180042:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6209e34100. 00010000:00000010:3.1:1775773811.180054:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202fc1400. 00010000:00000010:3.1:1775773811.180058:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62262abe00. 00010000:00000010:3.1:1775773811.180061:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62262a9e00. 00010000:00000010:3.1:1775773811.180064:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62262ab900. 00010000:00000010:3.1:1775773811.180067:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335d8000. 00010000:00000010:3.1:1775773811.180069:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62262ab700. 00010000:00000010:3.1:1775773811.180072:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca47000. 00010000:00000010:3.1:1775773811.180075:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c46fd00. 00010000:00000010:3.1:1775773811.180078:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335d8a00. 00010000:00000010:3.1:1775773811.180081:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62262aba00. 00010000:00000010:3.1:1775773811.180084:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335d9400. 00010000:00000010:3.1:1775773811.180087:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62262ab600. 00010000:00000010:3.1:1775773811.180089:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335d9c00. 00010000:00000010:3.1:1775773811.180092:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62262abf00. 00010000:00000010:3.1:1775773811.180094:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335d8c00. 00010000:00000010:3.1:1775773811.180095:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c46f400. 00010000:00000010:3.1:1775773811.180097:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335d9600. 00010000:00000010:3.1:1775773811.180104:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62262ab000. 00010000:00000010:3.1:1775773811.180106:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62082ce400. 00010000:00000010:3.1:1775773811.180111:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62262ab500. 00010000:00000010:3.1:1775773811.180113:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202fc0200. 00010000:00000010:3.1:1775773811.180114:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62262abb00. 00010000:00000010:3.1:1775773811.180118:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202fc1000. 00010000:00000010:3.1:1775773811.180123:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c46f800. 00010000:00000010:3.1:1775773811.180127:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622ca47e00. 00010000:00000010:1.1:1775773820.392800:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e320c00. 00010000:00000010:1.1:1775773820.392804:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62264d7700. 00010000:00000010:1.1:1775773820.392807:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c55c400. 00010000:00000010:1.1:1775773820.392811:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e3a00. 00010000:00000010:1.1:1775773820.392814:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c55d200. 00010000:00000010:1.1:1775773820.392817:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa5d00. 00010000:00000010:1.1:1775773820.392819:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621836b800. 00010000:00000010:1.1:1775773820.392821:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62264d7400. 00010000:00000010:1.1:1775773820.392823:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621836ae00. 00010000:00000010:1.1:1775773820.392825:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62264d7800. 00010000:00000010:1.1:1775773820.392826:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621836b600. 00010000:00000010:1.1:1775773820.392828:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e3100. 00010000:00000010:1.1:1775773820.392830:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e321600. 00010000:00000010:1.1:1775773820.392832:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e3600. 00010000:00000010:1.1:1775773820.392833:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e320e00. 00010000:00000010:1.1:1775773820.392835:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa5400. 00010000:00000010:1.1:1775773820.392837:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d2e00. 00010000:00000010:1.1:1775773820.392839:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e322400. 00010000:00000010:1.1:1775773820.392841:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d3400. 00010000:00000010:1.1:1775773820.392842:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e3f00. 00010000:00000010:1.1:1775773820.392844:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e320200. 00010000:00000010:1.1:1775773820.392846:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e322800. 00010000:00000010:1.1:1775773820.392848:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621836bc00. 00010000:00000010:1.1:1775773820.392849:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa5800. 00010000:00000010:1.1:1775773820.392851:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d2c00. 00010000:00000010:1.1:1775773820.392853:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e3000. 00010000:00000010:1.1:1775773820.392854:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e321a00. 00010000:00000010:1.1:1775773820.392856:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e3500. 00010000:00000010:1.1:1775773820.392858:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e321000. 00010000:00000010:1.1:1775773820.392860:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e3b00. 00010000:00000010:1.1:1775773820.392862:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e321e00. 00010000:00000010:1.1:1775773820.392864:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e3c00. 00010000:00000010:1.1:1775773820.392865:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e321200. 00010000:00000010:1.1:1775773820.392867:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa5e00. 00010000:00000010:1.1:1775773820.392870:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd3a000. 00010000:00000010:1.1:1775773820.392872:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62264d7e00. 00010000:00000010:1.1:1775773820.392874:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621836a000. 00010000:00000010:1.1:1775773820.392875:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e3300. 00010000:00000010:1.1:1775773820.392877:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd3a600. 00010000:00000010:1.1:1775773820.392878:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e322e00. 00010000:00000010:1.1:1775773820.392880:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621836a400. 00010000:00000010:1.1:1775773820.392882:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e322900. 00010000:00000010:1.1:1775773820.392884:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd3ba00. 00010000:00000010:1.1:1775773820.392886:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e322700. 00010000:00000010:1.1:1775773820.392887:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6218368e00. 00010000:00000010:1.1:1775773820.392889:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa5900. 00010000:00000010:1.1:1775773820.392890:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d2600. 00010000:00000010:1.1:1775773820.392892:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62264d7900. 00010000:00000010:1.1:1775773820.392894:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621836be00. 00010000:00000010:1.1:1775773820.392895:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e3200. 00010000:00000010:1.1:1775773820.392897:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd3b000. 00010000:00000010:1.1:1775773820.392898:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e3d00. 00010000:00000010:1.1:1775773820.392900:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d3c00. 00010000:00000010:1.1:1775773820.392902:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e3400. 00010000:00000010:1.1:1775773820.392903:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd3be00. 00010000:00000010:1.1:1775773820.392905:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e3800. 00010000:00000010:1.1:1775773820.392906:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd3b200. 00010000:00000010:1.1:1775773820.392909:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e41a500. 00010000:00000010:1.1:1775773820.392911:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd3a400. 00010000:00000010:1.1:1775773820.392913:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e3e00. 00010000:00000010:1.1:1775773820.392914:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd3a800. 00010000:00000010:1.1:1775773820.392916:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e3900. 00010000:00000010:1.1:1775773820.392918:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd3b800. 00010000:00000010:1.1:1775773820.392920:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e3700. 00010000:00000010:1.1:1775773820.392921:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d3000. 00010000:00000010:1.1:1775773820.392923:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa5700. 00010000:00000010:1.1:1775773820.392925:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d3e00. 00010000:00000010:1.1:1775773820.392927:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa5a00. 00010000:00000010:1.1:1775773820.392929:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d3a00. 00010000:00000010:1.1:1775773820.392931:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e41ab00. 00010000:00000010:1.1:1775773820.392933:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6218369a00. 00010000:00000010:1.1:1775773820.392935:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111f00. 00010000:00000010:1.1:1775773820.392937:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd3aa00. 00010000:00000010:1.1:1775773820.392938:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e41ac00. 00010000:00000010:1.1:1775773820.392940:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd3b400. 00010000:00000010:1.1:1775773820.392942:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111000. 00010000:00000010:1.1:1775773820.392943:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd3bc00. 00010000:00000010:1.1:1775773820.392945:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e41a300. 00010000:00000010:1.1:1775773820.392946:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d2a00. 00010000:00000010:1.1:1775773820.392948:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111500. 00010000:00000010:1.1:1775773820.392950:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d2000. 00010000:00000010:1.1:1775773820.392951:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62264d7a00. 00010000:00000010:1.1:1775773820.392953:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd3ac00. 00010000:00000010:1.1:1775773820.392954:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62264d7100. 00010000:00000010:1.1:1775773820.392956:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621836a800. 00010000:00000010:1.1:1775773820.392958:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111b00. 00010000:00000010:1.1:1775773820.392959:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd3b600. 00010000:00000010:1.1:1775773820.392961:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e41ad00. 00010000:00000010:1.1:1775773820.392962:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd3ae00. 00010000:00000010:1.1:1775773820.392964:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111c00. 00010000:00000010:1.1:1775773820.392965:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd3a200. 00010000:00000010:1.1:1775773820.392967:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa5100. 00010000:00000010:1.1:1775773820.392968:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621836ac00. 00010000:00000010:1.1:1775773820.392970:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111300. 00010000:00000010:1.1:1775773820.392972:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202f58400. 00010000:00000010:1.1:1775773820.392974:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111200. 00010000:00000010:1.1:1775773820.392976:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6218369000. 00010000:00000010:1.1:1775773820.392978:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111d00. 00010000:00000010:1.1:1775773820.392980:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621836b400. 00010000:00000010:1.1:1775773820.392982:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e41a200. 00010000:00000010:1.1:1775773820.392984:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6218369e00. 00010000:00000010:1.1:1775773820.392985:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e41a400. 00010000:00000010:1.1:1775773820.392987:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6218369200. 00010000:00000010:1.1:1775773820.392989:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111400. 00010000:00000010:1.1:1775773820.392990:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202f58800. 00010000:00000010:1.1:1775773820.392992:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111800. 00010000:00000010:1.1:1775773820.392994:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202f59800. 00010000:00000010:1.1:1775773820.392996:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa5600. 00010000:00000010:1.1:1775773820.392998:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d2400. 00010000:00000010:1.1:1775773820.392999:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111e00. 00010000:00000010:1.1:1775773820.393001:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d3200. 00010000:00000010:1.1:1775773820.393002:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62264d7600. 00010000:00000010:1.1:1775773820.393004:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d2200. 00010000:00000010:1.1:1775773820.393005:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e41a800. 00010000:00000010:1.1:1775773820.393007:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6218368400. 00010000:00000010:1.1:1775773820.393009:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62264d7f00. 00010000:00000010:1.1:1775773820.393010:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202f58600. 00010000:00000010:1.1:1775773820.393012:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62264d7000. 00010000:00000010:1.1:1775773820.393013:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210407200. 00010000:00000010:1.1:1775773820.393015:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111900. 00010000:00000010:1.1:1775773820.393017:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d3600. 00010000:00000010:1.1:1775773820.393018:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111700. 00010000:00000010:1.1:1775773820.393020:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202f58000. 00010000:00000010:1.1:1775773820.393021:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111a00. 00010000:00000010:1.1:1775773820.393023:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202f58a00. 00010000:00000010:1.1:1775773820.393024:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62264d7500. 00010000:00000010:1.1:1775773820.393026:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210406000. 00010000:00000010:1.1:1775773820.393028:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62264d7b00. 00010000:00000010:1.1:1775773820.393029:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210406200. 00010000:00000010:1.1:1775773820.393031:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62264d7c00. 00010000:00000010:1.1:1775773820.393033:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d2800. 00010000:00000010:1.1:1775773820.393035:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111100. 00010000:00000010:1.1:1775773820.393037:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202f59400. 00010000:00000010:1.1:1775773820.393038:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622c111600. 00010000:00000010:1.1:1775773820.393040:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202f59c00. 00010000:00000010:1.1:1775773820.393042:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e41ae00. 00010000:00000010:1.1:1775773820.393044:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d863c00. 00010000:00000010:1.1:1775773820.393045:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e41a900. 00010000:00000010:1.1:1775773820.393047:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d862c00. 00010000:00000010:1.1:1775773820.393049:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e41a700. 00010000:00000010:1.1:1775773820.393050:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d863600. 00010000:00000010:1.1:1775773820.393052:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623459c300. 00010000:00000010:1.1:1775773820.393054:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202f58c00. 00010000:00000010:1.1:1775773820.393056:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e41aa00. 00010000:00000010:1.1:1775773820.393057:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210407a00. 00010000:00000010:1.1:1775773820.393059:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623459c200. 00010000:00000010:1.1:1775773820.393060:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210407000. 00010000:00000010:1.1:1775773820.393062:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623459cd00. 00010000:00000010:1.1:1775773820.393064:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202f59600. 00010000:00000010:1.1:1775773820.393065:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e41a100. 00010000:00000010:1.1:1775773820.393067:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210406400. 00010000:00000010:1.1:1775773820.393068:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62264d7300. 00010000:00000010:1.1:1775773820.393070:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d862e00. 00010000:00000010:1.1:1775773820.393071:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623459c400. 00010000:00000010:1.1:1775773820.393073:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202f58e00. 00010000:00000010:1.1:1775773820.393075:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e41a600. 00010000:00000010:1.1:1775773820.393077:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202f58200. 00010000:00000010:1.1:1775773820.393079:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623459c800. 00010000:00000010:1.1:1775773820.393080:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202f59a00. 00010000:00000010:1.1:1775773820.393082:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62264d7200. 00010000:00000010:1.1:1775773820.393084:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210406c00. 00010000:00000010:1.1:1775773820.393086:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623459ce00. 00010000:00000010:1.1:1775773820.393088:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202f59000. 00010000:00000010:1.1:1775773820.393090:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623459c900. 00010000:00000010:1.1:1775773820.393092:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202f59e00. 00010000:00000010:1.1:1775773820.393093:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623459c700. 00010000:00000010:1.1:1775773820.393095:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202f59200. 00010000:00000010:1.1:1775773820.393097:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622bfa5f00. 00010000:00000010:1.1:1775773820.393098:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62057d3800. 00010000:00000010:1.1:1775773820.393100:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623459ca00. 00010000:00000010:1.1:1775773820.393102:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddb6a00. 00010000:00000010:1.1:1775773820.393103:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623459c100. 00010000:00000010:1.1:1775773820.393105:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddb7400. 00010000:00000010:1.1:1775773820.393106:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623459c600. 00010000:00000010:1.1:1775773820.393108:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddb7c00. 00010000:00000010:1.1:1775773820.393110:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623459cf00. 00010000:00000010:1.1:1775773820.393111:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddb6c00. 00010000:00000010:1.1:1775773820.393113:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623459c000. 00010000:00000010:1.1:1775773820.393114:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddb7600. 00010000:00000010:1.1:1775773820.393116:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623459c500. 00010000:00000010:1.1:1775773820.393117:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddb6e00. 00010000:00000010:1.1:1775773820.393120:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623459cb00. 00010000:00000010:1.1:1775773820.393122:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6210407c00. 00010000:00000010:1.1:1775773820.393124:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e41af00. 00010000:00000010:1.1:1775773820.393127:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddb6200. 00010000:00000010:1.1:1775773820.393129:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623459cc00. 00010000:00000010:1.1:1775773820.393132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddb7a00. 00010000:00000010:1.1:1775773820.393151:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234418800. 00010000:00000010:1.1:1775773820.393154:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddb7000. 00010000:00000010:1.1:1775773820.393156:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234418e00. 00010000:00000010:1.1:1775773820.393158:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddb7e00. 00010000:00000010:1.1:1775773820.393160:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e41a000. 00010000:00000010:1.1:1775773820.393162:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e281600. 00010000:00000010:1.1:1775773820.393165:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207a64100. 00010000:00000010:1.1:1775773820.393167:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddb7200. 00010000:00000010:1.1:1775773820.393169:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234418900. 00010000:00000010:1.1:1775773820.393172:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddb6400. 00010000:00000010:1.1:1775773820.393174:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207a64600. 00010000:00000010:1.1:1775773820.393176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddb6800. 00010000:00000010:1.1:1775773820.393178:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234418700. 00010000:00000010:1.1:1775773820.393180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621ddb7800. 00010000:00000010:1.1:1775773820.393182:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234418a00. 00010000:00000010:1.1:1775773820.393185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e280a00. 00000800:00000200:1.1:1775773825.339726:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773825.339729:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775773825.340052:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773825.340054:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775773825.340289:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773825.340291:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775773830.459722:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773830.459727:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775773830.460155:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773830.460157:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775773830.460383:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773830.460384:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000080:00000010:1.1:1775773830.634168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6200025760. 00000080:00000010:1.1:1775773830.634178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8720. 00000080:00000010:1.1:1775773830.634184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4360. 00000080:00000010:1.1:1775773830.638171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4400. 00000080:00000010:1.1:1775773830.638179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fdc00. 00000080:00000010:1.1:1775773830.638185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62120c49e0. 00000080:00000010:1.1:1775773830.638189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4080. 00000080:00000010:1.1:1775773830.638193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4b20. 00000080:00000010:1.1:1775773830.638196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab43e0. 00000080:00000010:1.1:1775773830.638199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab44e0. 00000080:00000010:1.1:1775773830.638202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4840. 00000080:00000010:1.1:1775773830.638205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4320. 00000080:00000010:1.1:1775773830.638211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8260. 00000080:00000010:1.1:1775773830.638213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6211bdb360. 00000080:00000010:1.1:1775773830.638218:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15b80. 00000080:00000010:1.1:1775773830.638221:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4860. 00000080:00000010:1.1:1775773830.638224:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4660. 00000080:00000010:1.1:1775773830.638230:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4240. 00000080:00000010:1.1:1775773830.638232:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4e60. 00000080:00000010:1.1:1775773830.638235:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4f20. 00000080:00000010:1.1:1775773830.638238:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c258c0. 00000080:00000010:1.1:1775773830.638242:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8b20. 00000080:00000010:1.1:1775773830.638245:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25fe0. 00000080:00000010:1.1:1775773830.638249:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c251e0. 00000080:00000010:1.1:1775773830.638252:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab47e0. 00000080:00000010:1.1:1775773830.638255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab47a0. 00000080:00000010:1.1:1775773830.638258:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab48e0. 00000080:00000010:1.1:1775773830.638262:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4120. 00000080:00000010:1.1:1775773830.638264:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8b80. 00000080:00000010:1.1:1775773830.638267:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4da0. 00000080:00000010:1.1:1775773830.638270:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab43c0. 00000080:00000010:1.1:1775773830.638273:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4200. 00000080:00000010:1.1:1775773830.638275:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4ba0. 00000080:00000010:1.1:1775773830.638278:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8180. 00000080:00000010:1.1:1775773830.638281:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a80e0. 00000080:00000010:1.1:1775773830.638284:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab40e0. 00000080:00000010:1.1:1775773830.638286:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15cc0. 00000080:00000010:1.1:1775773830.638290:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab45c0. 00000080:00000010:1.1:1775773830.638292:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a83c0. 00000080:00000010:1.1:1775773830.638295:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab48c0. 00000080:00000010:1.1:1775773830.638297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8120. 00000080:00000010:1.1:1775773830.638300:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a86c0. 00000080:00000010:1.1:1775773830.638302:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4c60. 00000080:00000010:1.1:1775773830.638305:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4bc0. 00000080:00000010:1.1:1775773830.638308:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4980. 00000080:00000010:1.1:1775773830.638310:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4180. 00000080:00000010:1.1:1775773830.638313:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4d00. 00000080:00000010:1.1:1775773830.638315:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4f80. 00000080:00000010:1.1:1775773830.638320:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab46a0. 00000080:00000010:1.1:1775773830.638325:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15220. 00000080:00000010:1.1:1775773830.638329:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25de0. 00000080:00000010:1.1:1775773830.638333:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c251a0. 00000080:00000010:1.1:1775773830.638337:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25660. 00000080:00000010:1.1:1775773830.638341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25320. 00000080:00000010:1.1:1775773830.638345:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15960. 00000080:00000010:1.1:1775773830.638349:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25da0. 00000080:00000010:1.1:1775773830.638353:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4820. 00000080:00000010:1.1:1775773830.638358:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4ea0. 00000080:00000010:1.1:1775773830.638362:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8cc0. 00000080:00000010:1.1:1775773830.638367:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4e20. 00000080:00000010:1.1:1775773830.638371:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4c20. 00000080:00000010:1.1:1775773830.638375:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a151a0. 00000080:00000010:1.1:1775773830.638379:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4b80. 00000080:00000010:1.1:1775773830.638383:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4f40. 00000080:00000010:1.1:1775773830.638387:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8c40. 00000080:00000010:1.1:1775773830.638392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c259a0. 00000080:00000010:1.1:1775773830.638396:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4be0. 00000080:00000010:1.1:1775773830.638400:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15e00. 00000080:00000010:1.1:1775773830.638405:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4f60. 00000080:00000010:1.1:1775773830.638409:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25260. 00000080:00000010:1.1:1775773830.638413:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab46c0. 00000080:00000010:1.1:1775773830.638417:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25340. 00000080:00000010:1.1:1775773830.638421:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25940. 00000080:00000010:1.1:1775773830.638425:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25180. 00000080:00000010:1.1:1775773830.638429:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25c80. 00000080:00000010:1.1:1775773830.638432:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4880. 00000080:00000010:1.1:1775773830.638436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4260. 00000080:00000010:1.1:1775773830.638441:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c254e0. 00000080:00000010:1.1:1775773830.638445:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8540. 00000080:00000010:1.1:1775773830.638450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15d80. 00000080:00000010:1.1:1775773830.638454:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4160. 00000080:00000010:1.1:1775773830.638458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4a20. 00000080:00000010:1.1:1775773830.638462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4940. 00000080:00000010:1.1:1775773830.638466:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4560. 00000080:00000010:1.1:1775773830.638469:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a159c0. 00000080:00000010:1.1:1775773830.638474:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15100. 00000080:00000010:1.1:1775773830.638478:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15e40. 00000080:00000010:1.1:1775773830.638482:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c254a0. 00000080:00000010:1.1:1775773830.638485:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4ae0. 00000080:00000010:1.1:1775773830.638489:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4720. 00000080:00000010:1.1:1775773830.638494:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a153a0. 00000080:00000010:1.1:1775773830.638497:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8de0. 00000080:00000010:1.1:1775773830.638501:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a87c0. 00000080:00000010:1.1:1775773830.638505:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4de0. 00000080:00000010:1.1:1775773830.638508:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15d40. 00000080:00000010:1.1:1775773830.638512:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4b40. 00000080:00000010:1.1:1775773830.638516:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab42e0. 00000080:00000010:1.1:1775773830.638520:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab48a0. 00000080:00000010:1.1:1775773830.638524:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15580. 00000080:00000010:1.1:1775773830.638527:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab49c0. 00000080:00000010:1.1:1775773830.638531:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4300. 00000080:00000010:1.1:1775773830.638535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25480. 00000080:00000010:1.1:1775773830.638539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4440. 00000080:00000010:1.1:1775773830.638543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15e60. 00000080:00000010:1.1:1775773830.638547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25580. 00000080:00000010:1.1:1775773830.638551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4a80. 00000080:00000010:1.1:1775773830.638555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4520. 00000080:00000010:1.1:1775773830.638559:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8be0. 00000080:00000010:1.1:1775773830.638563:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4d60. 00000080:00000010:1.1:1775773830.638567:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4fa0. 00000080:00000010:1.1:1775773830.638571:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4640. 00000080:00000010:1.1:1775773830.638574:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a151e0. 00000080:00000010:1.1:1775773830.638578:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25ce0. 00000080:00000010:1.1:1775773830.638582:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a88c0. 00000080:00000010:1.1:1775773830.638586:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8b00. 00000080:00000010:1.1:1775773830.638591:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c252a0. 00000080:00000010:1.1:1775773830.638594:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25c40. 00000080:00000010:1.1:1775773830.638598:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a150c0. 00000080:00000010:1.1:1775773830.638602:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15120. 00000080:00000010:1.1:1775773830.638605:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15c20. 00000080:00000010:1.1:1775773830.638609:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4a40. 00000080:00000010:1.1:1775773830.638613:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4aa0. 00000080:00000010:1.1:1775773830.638617:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab43a0. 00000080:00000010:1.1:1775773830.638620:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4c80. 00000080:00000010:1.1:1775773830.638624:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab47c0. 00000080:00000010:1.1:1775773830.638628:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4e00. 00000080:00000010:1.1:1775773830.638631:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4580. 00000080:00000010:1.1:1775773830.638635:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4fe0. 00000080:00000010:1.1:1775773830.638638:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8740. 00000080:00000010:1.1:1775773830.638641:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15840. 00000080:00000010:1.1:1775773830.638645:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8560. 00000080:00000010:1.1:1775773830.638649:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a84c0. 00000080:00000010:1.1:1775773830.638652:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4780. 00000080:00000010:1.1:1775773830.638656:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c253a0. 00000080:00000010:1.1:1775773830.638659:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210ab4040. 00000080:00000010:1.1:1775773830.638663:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25a40. 00010000:00000010:1.1:1775773830.638668:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62346e6700. 00010000:00000010:1.1:1775773830.638672:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b49600. 00010000:00000010:1.1:1775773830.638676:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202fc3c00. 00010000:00000010:1.1:1775773830.638680:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5c9800. 00010000:00000010:1.1:1775773830.638682:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62346e6a00. 00010000:00000010:1.1:1775773830.638685:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b48e00. 00010000:00000010:1.1:1775773830.638688:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62346e6100. 00010000:00000010:1.1:1775773830.638692:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622647ea00. 00010000:00000010:1.1:1775773830.638695:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62346e6600. 00010000:00000010:1.1:1775773830.638698:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622647f400. 00010000:00000010:1.1:1775773830.638701:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62346e6f00. 00010000:00000010:1.1:1775773830.638704:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622647fc00. 00010000:00000010:1.1:1775773830.638706:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62346e6000. 00010000:00000010:1.1:1775773830.638710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622647ec00. 00010000:00000010:1.1:1775773830.638712:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62346e6500. 00010000:00000010:1.1:1775773830.638715:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622647f600. 00010000:00000010:1.1:1775773830.638718:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62346e6b00. 00010000:00000010:1.1:1775773830.638720:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622647ee00. 00010000:00000010:1.1:1775773830.638723:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62346e6c00. 00010000:00000010:1.1:1775773830.638725:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622647e200. 00010000:00000010:1.1:1775773830.638727:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62346e6300. 00010000:00000010:1.1:1775773830.638729:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622647fa00. 00010000:00000010:1.1:1775773830.638732:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62346e6200. 00010000:00000010:1.1:1775773830.638734:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622647f000. 00010000:00000010:1.1:1775773830.638737:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202fc3300. 00010000:00000010:1.1:1775773830.638739:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5c8600. 00010000:00000010:1.1:1775773830.638742:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623491e000. 00010000:00000010:1.1:1775773830.638745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc4ba00. 00010000:00000010:1.1:1775773830.638748:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202fc3200. 00010000:00000010:1.1:1775773830.638750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc4a200. 00010000:00000010:1.1:1775773830.638753:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62346e6d00. 00010000:00000010:1.1:1775773830.638756:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622647fe00. 00010000:00000010:1.1:1775773830.638759:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62346e6400. 00010000:00000010:1.1:1775773830.638761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622647f200. 00010000:00000010:1.1:1775773830.638765:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207de3f00. 00010000:00000010:1.1:1775773830.638768:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622647e400. 00010000:00000010:1.1:1775773830.638771:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207de3000. 00010000:00000010:1.1:1775773830.638774:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622647e800. 00010000:00000010:1.1:1775773830.638777:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207de3500. 00010000:00000010:1.1:1775773830.638779:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622647f800. 00010000:00000010:1.1:1775773830.638782:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207947700. 00010000:00000010:1.1:1775773830.638785:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62339f1800. 00010000:00000010:1.1:1775773830.638788:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202fc3d00. 00010000:00000010:1.1:1775773830.638791:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5c8000. 00010000:00000010:1.1:1775773830.638794:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207de3b00. 00010000:00000010:1.1:1775773830.638797:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62339f1c00. 00010000:00000010:1.1:1775773830.638800:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207947a00. 00010000:00000010:1.1:1775773830.638803:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62339f0800. 00010000:00000010:1.1:1775773830.638806:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207de3c00. 00010000:00000010:1.1:1775773830.638809:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622647e600. 00010000:00000010:1.1:1775773830.638813:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207de3300. 00010000:00000010:1.1:1775773830.638815:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622647e000. 00010000:00000010:1.1:1775773830.638818:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207de3200. 00010000:00000010:1.1:1775773830.638822:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233986600. 00010000:00000010:1.1:1775773830.638825:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207de3d00. 00010000:00000010:1.1:1775773830.638828:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233987000. 00010000:00000010:1.1:1775773830.638831:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202fc3400. 00010000:00000010:1.1:1775773830.638834:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5c8a00. 00010000:00000010:1.1:1775773830.638836:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202fc3800. 00010000:00000010:1.1:1775773830.638839:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233987e00. 00010000:00000010:1.1:1775773830.638842:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207de3400. 00010000:00000010:1.1:1775773830.638844:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233987200. 00010000:00000010:1.1:1775773830.638847:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207de3800. 00010000:00000010:1.1:1775773830.638850:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233986400. 00010000:00000010:1.1:1775773830.638853:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207de3e00. 00010000:00000010:1.1:1775773830.638855:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233986800. 00010000:00000010:1.1:1775773830.638858:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202fc3e00. 00010000:00000010:1.1:1775773830.638861:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5c9400. 00010000:00000010:1.1:1775773830.638864:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202fc3900. 00010000:00000010:1.1:1775773830.638866:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5c9c00. 00010000:00000010:1.1:1775773830.638869:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207de3900. 00010000:00000010:1.1:1775773830.638872:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233987800. 00010000:00000010:1.1:1775773830.638875:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623491e500. 00010000:00000010:1.1:1775773830.638878:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc4aa00. 00010000:00000010:1.1:1775773830.638881:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207de3700. 00010000:00000010:1.1:1775773830.638884:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233986000. 00010000:00000010:1.1:1775773830.638887:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202fc3700. 00010000:00000010:1.1:1775773830.638891:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5c8c00. 00010000:00000010:1.1:1775773830.638893:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207de3a00. 00010000:00000010:1.1:1775773830.638896:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233986a00. 00010000:00000010:1.1:1775773830.638899:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207de3100. 00010000:00000010:1.1:1775773830.638901:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233987400. 00010000:00000010:1.1:1775773830.638905:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207947600. 00010000:00000010:1.1:1775773830.638907:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5c9600. 00010000:00000010:1.1:1775773830.638910:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207de3600. 00010000:00000010:1.1:1775773830.638913:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233987c00. 00010000:00000010:1.1:1775773830.638916:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d80c00. 00010000:00000010:1.1:1775773830.638919:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233986c00. 00010000:00000010:1.1:1775773830.638922:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d80300. 00010000:00000010:1.1:1775773830.638924:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233987600. 00010000:00000010:1.1:1775773830.638927:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d80200. 00010000:00000010:1.1:1775773830.638930:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc4a600. 00010000:00000010:1.1:1775773830.638933:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207947f00. 00010000:00000010:1.1:1775773830.638936:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233986e00. 00010000:00000010:1.1:1775773830.638939:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d80d00. 00010000:00000010:1.1:1775773830.638941:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233986200. 00010000:00000010:1.1:1775773830.638944:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d80400. 00010000:00000010:1.1:1775773830.638947:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233987a00. 00010000:00000010:1.1:1775773830.638949:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623491eb00. 00010000:00000010:1.1:1775773830.638952:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc4ae00. 00010000:00000010:1.1:1775773830.638955:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207947000. 00010000:00000010:1.1:1775773830.638958:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62339f1200. 00010000:00000010:1.1:1775773830.638961:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207947500. 00010000:00000010:1.1:1775773830.638963:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62339f1e00. 00010000:00000010:1.1:1775773830.638967:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62267c7d00. 00010000:00000010:1.1:1775773830.638994:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62339f1000. 00010000:00000010:1.1:1775773830.638998:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62267c7400. 00010000:00000010:1.1:1775773830.639001:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62339f0200. 00010000:00000010:1.1:1775773830.639004:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207947b00. 00010000:00000010:1.1:1775773830.639022:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc4b200. 00010000:00000010:1.1:1775773830.639026:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d80800. 00010000:00000010:1.1:1775773830.639029:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62339f0400. 00010000:00000010:1.1:1775773830.639032:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d80e00. 00010000:00000010:1.1:1775773830.639035:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234ade400. 00010000:00000010:1.1:1775773830.639038:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62267c7800. 00010000:00000010:1.1:1775773830.639041:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234ade800. 00010000:00000010:1.1:1775773830.639044:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62267c7e00. 00010000:00000010:1.1:1775773830.639047:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5c8e00. 00010000:00000010:1.1:1775773830.639049:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d80900. 00010000:00000010:1.1:1775773830.639052:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234adf800. 00010000:00000010:1.1:1775773830.639055:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d80700. 00010000:00000010:1.1:1775773830.639058:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234ade600. 00010000:00000010:1.1:1775773830.639060:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623491ec00. 00010000:00000010:1.1:1775773830.639063:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc4a400. 00010000:00000010:1.1:1775773830.639064:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623491e300. 00010000:00000010:1.1:1775773830.639066:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234ade000. 00010000:00000010:1.1:1775773830.639068:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d80a00. 00010000:00000010:1.1:1775773830.639069:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234adea00. 00010000:00000010:1.1:1775773830.639071:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62267c7900. 00010000:00000010:1.1:1775773830.639072:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5c8200. 00010000:00000010:1.1:1775773830.639074:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207947c00. 00010000:00000010:1.1:1775773830.639076:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62339f0000. 00010000:00000010:1.1:1775773830.639077:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62267c7700. 00010000:00000010:1.1:1775773830.639079:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234adf400. 00010000:00000010:1.1:1775773830.639080:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207947300. 00010000:00000010:1.1:1775773830.639082:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc4b800. 00010000:00000010:1.1:1775773830.639083:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62267c7a00. 00010000:00000010:1.1:1775773830.639085:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5c9a00. 00010000:00000010:1.1:1775773830.639086:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207947200. 00010000:00000010:1.1:1775773830.639088:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62339f0600. 00010000:00000010:1.1:1775773830.639089:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d80100. 00010000:00000010:1.1:1775773830.639092:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234adfc00. 00010000:00000010:1.1:1775773830.639094:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207947d00. 00010000:00000010:1.1:1775773830.639095:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62339f0a00. 00010000:00000010:1.1:1775773830.639098:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226053400. 00010000:00000010:1.1:1775773830.639099:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62339f0e00. 00010000:00000010:1.1:1775773830.639101:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226053800. 00010000:00000010:1.1:1775773830.639103:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62339f1a00. 00010000:00000010:1.1:1775773830.639104:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d80600. 00010000:00000010:1.1:1775773830.639106:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c55dc00. 00010000:00000010:1.1:1775773830.639108:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d80f00. 00010000:00000010:1.1:1775773830.639109:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234adec00. 00010000:00000010:1.1:1775773830.639111:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d80000. 00010000:00000010:1.1:1775773830.639113:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234adf600. 00010000:00000010:1.1:1775773830.639114:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62267c7100. 00010000:00000010:1.1:1775773830.639116:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c55cc00. 00010000:00000010:1.1:1775773830.639118:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62267c7600. 00010000:00000010:1.1:1775773830.639119:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6218368200. 00010000:00000010:1.1:1775773830.639121:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623491e200. 00010000:00000010:1.1:1775773830.639123:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335de000. 00010000:00000010:1.1:1775773830.639125:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62267c7f00. 00010000:00000010:1.1:1775773830.639126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234adee00. 00010000:00000010:1.1:1775773830.639128:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d80500. 00010000:00000010:1.1:1775773830.639129:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234ade200. 00010000:00000010:1.1:1775773830.639131:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62267c7000. 00010000:00000010:1.1:1775773830.639132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234adfa00. 00010000:00000010:1.1:1775773830.639147:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d80b00. 00010000:00000010:1.1:1775773830.639152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234adf000. 00010000:00000010:1.1:1775773830.639154:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623491ed00. 00010000:00000010:1.1:1775773830.639156:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335dfc00. 00010000:00000010:1.1:1775773830.639159:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce9cd00. 00010000:00000010:1.1:1775773830.639161:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335de600. 00010000:00000010:1.1:1775773830.639162:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623491e400. 00010000:00000010:1.1:1775773830.639165:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335df400. 00010000:00000010:1.1:1775773830.639166:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce9c400. 00010000:00000010:1.1:1775773830.639168:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c55ca00. 00010000:00000010:1.1:1775773830.639170:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62267c7500. 00010000:00000010:1.1:1775773830.639172:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234adfe00. 00010000:00000010:1.1:1775773830.639174:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226053e00. 00010000:00000010:1.1:1775773830.639175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6234adf200. 00010000:00000010:1.1:1775773830.639177:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce9c800. 00010000:00000010:1.1:1775773830.639179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335df600. 00010000:00000010:1.1:1775773830.639180:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62267c7b00. 00010000:00000010:1.1:1775773830.639182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6218368800. 00010000:00000010:1.1:1775773830.639183:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62267c7c00. 00010000:00000010:1.1:1775773830.639185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6218369800. 00010000:00000010:1.1:1775773830.639187:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce9ce00. 00010000:00000010:1.1:1775773830.639189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd39600. 00010000:00000010:1.1:1775773830.639212:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce9c900. 00010000:00000010:1.1:1775773830.639214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335dee00. 00010000:00000010:1.1:1775773830.639215:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce9c700. 00010000:00000010:1.1:1775773830.639217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd39e00. 00010000:00000010:1.1:1775773830.639219:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce9ca00. 00010000:00000010:1.1:1775773830.639220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd39200. 00010000:00000010:1.1:1775773830.639222:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62267c7300. 00010000:00000010:1.1:1775773830.639223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd38400. 00010000:00000010:1.1:1775773830.639225:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623491e800. 00010000:00000010:1.1:1775773830.639227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335de200. 00010000:00000010:1.1:1775773830.639228:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce9c100. 00010000:00000010:1.1:1775773830.639230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd38800. 00010000:00000010:1.1:1775773830.639232:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62267c7200. 00010000:00000010:1.1:1775773830.639233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd39800. 00010000:00000010:1.1:1775773830.639235:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226053900. 00010000:00000010:1.1:1775773830.639237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c55da00. 00010000:00000010:1.1:1775773830.639239:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce9c600. 00010000:00000010:1.1:1775773830.639241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd38600. 00010000:00000010:1.1:1775773830.639243:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226053700. 00010000:00000010:1.1:1775773830.639244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335dec00. 00010000:00000010:1.1:1775773830.639246:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226053a00. 00010000:00000010:1.1:1775773830.639248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c55c200. 00010000:00000010:1.1:1775773830.639250:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce9cf00. 00010000:00000010:1.1:1775773830.639251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd38000. 00010000:00000010:1.1:1775773830.639253:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce9c000. 00010000:00000010:1.1:1775773830.639254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd38a00. 00010000:00000010:1.1:1775773830.639256:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e322a00. 00010000:00000010:1.1:1775773830.639257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6218368600. 00010000:00000010:1.1:1775773830.639259:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce9c500. 00010000:00000010:1.1:1775773830.639261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd39400. 00010000:00000010:1.1:1775773830.639262:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce9cb00. 00010000:00000010:1.1:1775773830.639264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd39c00. 00010000:00000010:1.1:1775773830.639265:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce9cc00. 00010000:00000010:1.1:1775773830.639266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd38c00. 00010000:00000010:1.1:1775773830.639268:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226053100. 00010000:00000010:1.1:1775773830.639270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335df800. 00010000:00000010:1.1:1775773830.639271:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226053600. 00010000:00000010:1.1:1775773830.639273:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c55d000. 00010000:00000010:1.1:1775773830.639274:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e322100. 00010000:00000010:1.1:1775773830.639276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6218368000. 00010000:00000010:1.1:1775773830.639277:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e322600. 00010000:00000010:1.1:1775773830.639279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6218368a00. 00010000:00000010:1.1:1775773830.639280:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226053f00. 00010000:00000010:1.1:1775773830.639282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c55d600. 00010000:00000010:1.1:1775773830.639283:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226053000. 00010000:00000010:1.1:1775773830.639285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c55de00. 00010000:00000010:1.1:1775773830.639286:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623491ee00. 00010000:00000010:1.1:1775773830.639288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335dea00. 00010000:00000010:1.1:1775773830.639289:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226053500. 00010000:00000010:1.1:1775773830.639291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335dfe00. 00010000:00000010:1.1:1775773830.639293:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6226053b00. 00010000:00000010:1.1:1775773830.639297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335de800. 00010000:00000010:1.1:1775773830.639299:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e322000. 00010000:00000010:1.1:1775773830.639300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd38e00. 00010000:00000010:1.1:1775773830.639302:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce9c300. 00010000:00000010:1.1:1775773830.639304:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd38200. 00010000:00000010:1.1:1775773830.639305:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e322f00. 00010000:00000010:1.1:1775773830.639307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd39a00. 00010000:00000010:1.1:1775773830.639308:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce9c200. 00010000:00000010:1.1:1775773830.639310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621dd39000. 00010000:00000010:1.1:1775773830.639311:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62117d1300. 00010000:00000010:1.1:1775773830.639313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335e3400. 00010000:00000010:1.1:1775773830.639315:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62117d1200. 00010000:00000010:1.1:1775773830.639317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335e3c00. 00010000:00000010:1.1:1775773830.639318:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62117d1d00. 00010000:00000010:1.1:1775773830.639320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335e2c00. 00010000:00000010:1.1:1775773830.639321:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62117d1400. 00010000:00000010:1.1:1775773830.639323:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335e3600. 00010000:00000010:1.1:1775773830.639324:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e322500. 00010000:00000010:1.1:1775773830.639326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6218369400. 00010000:00000010:1.1:1775773830.639327:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62117d1800. 00010000:00000010:1.1:1775773830.639329:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335df200. 00010000:00000010:1.1:1775773830.639333:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e322b00. 00010000:00000010:1.1:1775773830.639335:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6218369c00. 00010000:00000010:1.1:1775773830.639336:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e322c00. 00010000:00000010:1.1:1775773830.639340:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6218368c00. 00010000:00000010:1.1:1775773830.639345:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62117d1e00. 00010000:00000010:1.1:1775773830.639346:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335e2e00. 00010000:00000010:1.1:1775773830.639348:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62117d1900. 00010000:00000010:1.1:1775773830.639350:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c55d800. 00010000:00000010:1.1:1775773830.639354:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62117d1700. 00010000:00000010:1.1:1775773830.639358:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62335e2200. 00000800:00000200:1.1:1775773835.579959:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773835.579969:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775773835.580568:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773835.580570:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775773835.580765:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773835.580767:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775773840.699789:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773840.699794:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775773840.700251:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773840.700254:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775773840.700553:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773840.700556:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000080:00000010:1.1:1775773840.911172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25000. 00000080:00000010:1.1:1775773840.911190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c76e0. 00000080:00000010:1.1:1775773840.915172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c72c0. 00000080:00000010:1.1:1775773840.915182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a89a0. 00000080:00000010:1.1:1775773840.915192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c72a0. 00000080:00000010:1.1:1775773840.915200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70dc0. 00000080:00000010:1.1:1775773840.915206:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7ce0. 00000080:00000010:1.1:1775773840.915212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7220. 00000080:00000010:1.1:1775773840.915216:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7740. 00000080:00000010:1.1:1775773840.915221:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7b60. 00000080:00000010:1.1:1775773840.915225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a84a0. 00000080:00000010:1.1:1775773840.915229:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c78c0. 00000080:00000010:1.1:1775773840.915234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7ca0. 00000080:00000010:1.1:1775773840.915238:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c78a0. 00000080:00000010:1.1:1775773840.915245:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7ea0. 00000080:00000010:1.1:1775773840.915249:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7fe0. 00000080:00000010:1.1:1775773840.915253:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7200. 00000080:00000010:1.1:1775773840.915258:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7ba0. 00000080:00000010:1.1:1775773840.915262:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7c20. 00000080:00000010:1.1:1775773840.915269:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7a80. 00000080:00000010:1.1:1775773840.915274:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7440. 00000080:00000010:1.1:1775773840.915278:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8480. 00000080:00000010:1.1:1775773840.915281:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8dc0. 00000080:00000010:1.1:1775773840.915286:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c74e0. 00000080:00000010:1.1:1775773840.915290:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70f60. 00000080:00000010:1.1:1775773840.915294:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8ae0. 00000080:00000010:1.1:1775773840.915298:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c78e0. 00000080:00000010:1.1:1775773840.915302:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c75c0. 00000080:00000010:1.1:1775773840.915307:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25d40. 00000080:00000010:1.1:1775773840.915311:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7980. 00000080:00000010:1.1:1775773840.915314:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70ec0. 00000080:00000010:1.1:1775773840.915318:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8fe0. 00000080:00000010:1.1:1775773840.915322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7400. 00000080:00000010:1.1:1775773840.915326:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df704e0. 00000080:00000010:1.1:1775773840.915330:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7a20. 00000080:00000010:1.1:1775773840.915335:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7940. 00000080:00000010:1.1:1775773840.915347:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25dc0. 00000080:00000010:1.1:1775773840.915351:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25280. 00000080:00000010:1.1:1775773840.915356:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a88a0. 00000080:00000010:1.1:1775773840.915361:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25ee0. 00000080:00000010:1.1:1775773840.915366:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a83e0. 00000080:00000010:1.1:1775773840.915371:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c75e0. 00000080:00000010:1.1:1775773840.915375:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7d60. 00000080:00000010:1.1:1775773840.915379:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7960. 00000080:00000010:1.1:1775773840.915383:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7ec0. 00000080:00000010:1.1:1775773840.915387:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7f20. 00000080:00000010:1.1:1775773840.915391:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7be0. 00000080:00000010:1.1:1775773840.915395:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7b20. 00000080:00000010:1.1:1775773840.915399:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c77e0. 00000080:00000010:1.1:1775773840.915403:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7e00. 00000080:00000010:1.1:1775773840.915406:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c79c0. 00000080:00000010:1.1:1775773840.915410:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25460. 00000080:00000010:1.1:1775773840.915414:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7bc0. 00000080:00000010:1.1:1775773840.915419:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70500. 00000080:00000010:1.1:1775773840.915422:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c73a0. 00000080:00000010:1.1:1775773840.915426:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8280. 00000080:00000010:1.1:1775773840.915431:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7000. 00000080:00000010:1.1:1775773840.915434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c77a0. 00000080:00000010:1.1:1775773840.915440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c79a0. 00000080:00000010:1.1:1775773840.915444:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c72e0. 00000080:00000010:1.1:1775773840.915449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7660. 00000080:00000010:1.1:1775773840.915453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25fa0. 00000080:00000010:1.1:1775773840.915457:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70320. 00000080:00000010:1.1:1775773840.915461:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25e00. 00000080:00000010:1.1:1775773840.915465:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25040. 00000080:00000010:1.1:1775773840.915468:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a85a0. 00000080:00000010:1.1:1775773840.915472:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8f80. 00000080:00000010:1.1:1775773840.915476:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a80c0. 00000080:00000010:1.1:1775773840.915480:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25980. 00000080:00000010:1.1:1775773840.915484:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c70e0. 00000080:00000010:1.1:1775773840.915488:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7120. 00000080:00000010:1.1:1775773840.915491:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25ea0. 00000080:00000010:1.1:1775773840.915495:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7b80. 00000080:00000010:1.1:1775773840.915499:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df700e0. 00000080:00000010:1.1:1775773840.915503:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7880. 00000080:00000010:1.1:1775773840.915507:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7580. 00000080:00000010:1.1:1775773840.915511:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7520. 00000080:00000010:1.1:1775773840.915515:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7a40. 00000080:00000010:1.1:1775773840.915519:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70680. 00000080:00000010:1.1:1775773840.915525:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7ac0. 00000080:00000010:1.1:1775773840.915529:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7da0. 00000080:00000010:1.1:1775773840.915532:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7c80. 00000080:00000010:1.1:1775773840.915536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7160. 00000080:00000010:1.1:1775773840.915540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7640. 00000080:00000010:1.1:1775773840.915545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d5c20. 00000080:00000010:1.1:1775773840.915551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62057fd9c0. 00000080:00000010:1.1:1775773840.915555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df706c0. 00000080:00000010:1.1:1775773840.915559:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8200. 00000080:00000010:1.1:1775773840.915566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc393e0. 00000080:00000010:1.1:1775773840.915571:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70f20. 00000080:00000010:1.1:1775773840.915575:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210aefe40. 00000080:00000010:1.1:1775773840.915580:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39120. 00000080:00000010:1.1:1775773840.915584:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210aef440. 00000080:00000010:1.1:1775773840.915588:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70180. 00000080:00000010:1.1:1775773840.915592:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39600. 00000080:00000010:1.1:1775773840.915596:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39180. 00000080:00000010:1.1:1775773840.915600:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc394c0. 00000080:00000010:1.1:1775773840.915604:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39c00. 00000080:00000010:1.1:1775773840.915608:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8b60. 00000080:00000010:1.1:1775773840.915612:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8240. 00000080:00000010:1.1:1775773840.915616:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39f20. 00000080:00000010:1.1:1775773840.915619:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70f00. 00000080:00000010:1.1:1775773840.915623:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df708c0. 00000080:00000010:1.1:1775773840.915627:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39e60. 00000080:00000010:1.1:1775773840.915631:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39820. 00000080:00000010:1.1:1775773840.915636:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc392c0. 00000080:00000010:1.1:1775773840.915640:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39b20. 00000080:00000010:1.1:1775773840.915644:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d5ce0. 00000080:00000010:1.1:1775773840.915648:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc390c0. 00000080:00000010:1.1:1775773840.915651:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc395c0. 00000080:00000010:1.1:1775773840.915655:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39980. 00000080:00000010:1.1:1775773840.915662:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06cfe0. 00000080:00000010:1.1:1775773840.915666:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39e00. 00000080:00000010:1.1:1775773840.915670:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39fa0. 00000080:00000010:1.1:1775773840.915674:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39300. 00000080:00000010:1.1:1775773840.915678:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39d00. 00000080:00000010:1.1:1775773840.915683:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39ec0. 00000080:00000010:1.1:1775773840.915687:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc393a0. 00000080:00000010:1.1:1775773840.915691:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc399c0. 00000080:00000010:1.1:1775773840.915695:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06c260. 00000080:00000010:1.1:1775773840.915699:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06c080. 00000080:00000010:1.1:1775773840.915703:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06ce20. 00000080:00000010:1.1:1775773840.915707:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc398e0. 00000080:00000010:1.1:1775773840.915711:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06c500. 00000080:00000010:1.1:1775773840.915714:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc394e0. 00000080:00000010:1.1:1775773840.915719:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39020. 00000080:00000010:1.1:1775773840.915722:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39e40. 00000080:00000010:1.1:1775773840.915726:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39b00. 00000080:00000010:1.1:1775773840.915730:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39920. 00000080:00000010:1.1:1775773840.915734:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39240. 00000080:00000010:1.1:1775773840.915737:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06c600. 00000080:00000010:1.1:1775773840.915741:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25ba0. 00000080:00000010:1.1:1775773840.915745:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8420. 00000080:00000010:1.1:1775773840.915749:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25f80. 00000080:00000010:1.1:1775773840.915753:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06c8c0. 00000080:00000010:1.1:1775773840.915757:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39b60. 00000080:00000010:1.1:1775773840.915760:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06c2c0. 00000080:00000010:1.1:1775773840.915764:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39500. 00000080:00000010:1.1:1775773840.915768:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39ea0. 00000080:00000010:1.1:1775773840.915772:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39aa0. 00000080:00000010:1.1:1775773840.915776:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc397e0. 00000080:00000010:1.1:1775773840.915779:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39320. 00000080:00000010:1.1:1775773840.915783:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06c400. 00000080:00000010:1.1:1775773840.915786:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc392e0. 00000080:00000010:1.1:1775773840.915790:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8e60. 00000080:00000010:1.1:1775773840.915794:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39be0. 00000080:00000010:1.1:1775773840.915797:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39ac0. 00000080:00000010:1.1:1775773840.915801:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc39200. 00000080:00000010:1.1:1775773840.915805:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621dc395a0. 00000080:00000010:1.1:1775773840.915809:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06c540. 00000080:00000010:1.1:1775773840.915813:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25b40. 00000080:00000010:1.1:1775773840.915818:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d52a0. 00000080:00000010:1.1:1775773840.915822:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f6a0. 00000080:00000010:1.1:1775773840.915825:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06c4e0. 00000080:00000010:1.1:1775773840.915843:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62060ecc80. 00000080:00000010:1.1:1775773840.915848:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d57a0. 00000080:00000010:1.1:1775773840.915851:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d59e0. 00000080:00000010:1.1:1775773840.915855:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06cda0. 00000080:00000010:1.1:1775773840.915858:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06c120. 00000080:00000010:1.1:1775773840.915862:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d5a20. 00000080:00000010:1.1:1775773840.915866:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d5840. 00000080:00000010:1.1:1775773840.915870:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d5b60. 00000080:00000010:1.1:1775773840.915875:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d5860. 00000080:00000010:1.1:1775773840.915878:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06cc00. 00000080:00000010:1.1:1775773840.915882:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d5460. 00000080:00000010:1.1:1775773840.915885:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d5580. 00000080:00000010:1.1:1775773840.915890:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d5900. 00000080:00000010:1.1:1775773840.915893:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d5920. 00000080:00000010:1.1:1775773840.915897:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d5c80. 00000080:00000010:1.1:1775773840.915901:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06cee0. 00000080:00000010:1.1:1775773840.915905:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25200. 00000080:00000010:1.1:1775773840.915908:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25b60. 00000080:00000010:1.1:1775773840.915911:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d5160. 00000080:00000010:1.1:1775773840.915915:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06c780. 00000080:00000010:1.1:1775773840.915919:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d5f60. 00000080:00000010:1.1:1775773840.915923:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25780. 00000080:00000010:1.1:1775773840.915926:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25640. 00000080:00000010:1.1:1775773840.915929:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d5ba0. 00000080:00000010:1.1:1775773840.915933:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25ae0. 00000080:00000010:1.1:1775773840.915936:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d5480. 00000080:00000010:1.1:1775773840.915940:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d52c0. 00000080:00000010:1.1:1775773840.915944:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d5fa0. 00000080:00000010:1.1:1775773840.915947:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d5120. 00000080:00000010:1.1:1775773840.915951:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620b0d5440. 00000080:00000010:1.1:1775773840.915954:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62060ece20. 00000080:00000010:1.1:1775773840.915957:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62060ec0c0. 00000080:00000010:1.1:1775773840.915960:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06cb00. 00000080:00000010:1.1:1775773840.915964:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62060ecea0. 00000080:00000010:1.1:1775773840.915968:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a81a0. 00000080:00000010:1.1:1775773840.915971:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62060ec500. 00010000:00000010:1.1:1775773840.915976:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb7400. 00010000:00000010:1.1:1775773840.915981:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225844a00. 00010000:00000010:1.1:1775773840.915985:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202abbc00. 00010000:00000010:1.1:1775773840.915989:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206206400. 00010000:00000010:1.1:1775773840.915992:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202abb300. 00010000:00000010:1.1:1775773840.915996:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206206800. 00010000:00000010:1.1:1775773840.916000:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de6b300. 00010000:00000010:1.1:1775773840.916003:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da42c00. 00010000:00000010:1.1:1775773840.916006:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202abb200. 00010000:00000010:1.1:1775773840.916009:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206207800. 00010000:00000010:1.1:1775773840.916012:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202abbd00. 00010000:00000010:1.1:1775773840.916016:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c1ece00. 00010000:00000010:1.1:1775773840.916019:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202abb400. 00010000:00000010:1.1:1775773840.916022:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206206600. 00010000:00000010:1.1:1775773840.916024:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202abb800. 00010000:00000010:1.1:1775773840.916026:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206206000. 00010000:00000010:1.1:1775773840.916029:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202abbe00. 00010000:00000010:1.1:1775773840.916032:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206206a00. 00010000:00000010:1.1:1775773840.916035:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202abb900. 00010000:00000010:1.1:1775773840.916037:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206207400. 00010000:00000010:1.1:1775773840.916040:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202abb700. 00010000:00000010:1.1:1775773840.916043:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da43600. 00010000:00000010:1.1:1775773840.916046:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202abba00. 00010000:00000010:1.1:1775773840.916049:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206207c00. 00010000:00000010:1.1:1775773840.916051:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202abb100. 00010000:00000010:1.1:1775773840.916054:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206206c00. 00010000:00000010:1.1:1775773840.916057:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202abb600. 00010000:00000010:1.1:1775773840.916059:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206207600. 00010000:00000010:1.1:1775773840.916062:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de6b200. 00010000:00000010:1.1:1775773840.916065:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206206e00. 00010000:00000010:1.1:1775773840.916068:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202abbf00. 00010000:00000010:1.1:1775773840.916071:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206206200. 00010000:00000010:1.1:1775773840.916073:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202abb000. 00010000:00000010:1.1:1775773840.916076:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206179800. 00010000:00000010:1.1:1775773840.916080:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d37200. 00010000:00000010:1.1:1775773840.916084:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206178600. 00010000:00000010:1.1:1775773840.916087:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d37d00. 00010000:00000010:1.1:1775773840.916090:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206178000. 00010000:00000010:1.1:1775773840.916093:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d37400. 00010000:00000010:1.1:1775773840.916095:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206178a00. 00010000:00000010:1.1:1775773840.916098:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d37800. 00010000:00000010:1.1:1775773840.916101:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206179400. 00010000:00000010:1.1:1775773840.916105:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234add900. 00010000:00000010:1.1:1775773840.916108:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da42e00. 00010000:00000010:1.1:1775773840.916111:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234add700. 00010000:00000010:1.1:1775773840.916114:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da42200. 00010000:00000010:1.1:1775773840.916117:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234adda00. 00010000:00000010:1.1:1775773840.916119:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206179c00. 00010000:00000010:1.1:1775773840.916122:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb7800. 00010000:00000010:1.1:1775773840.916124:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c1edc00. 00010000:00000010:1.1:1775773840.916127:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234add100. 00010000:00000010:1.1:1775773840.916130:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da43a00. 00010000:00000010:1.1:1775773840.916147:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234add600. 00010000:00000010:1.1:1775773840.916151:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206178c00. 00010000:00000010:1.1:1775773840.916154:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d37e00. 00010000:00000010:1.1:1775773840.916157:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206179600. 00010000:00000010:1.1:1775773840.916160:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb7e00. 00010000:00000010:1.1:1775773840.916163:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202d35e00. 00010000:00000010:1.1:1775773840.916166:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d37900. 00010000:00000010:1.1:1775773840.916169:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206178e00. 00010000:00000010:1.1:1775773840.916172:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d37700. 00010000:00000010:1.1:1775773840.916175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c1ec000. 00010000:00000010:1.1:1775773840.916178:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cca4200. 00010000:00000010:1.1:1775773840.916180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da43000. 00010000:00000010:1.1:1775773840.916184:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d37a00. 00010000:00000010:1.1:1775773840.916187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206178200. 00010000:00000010:1.1:1775773840.916190:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cca4d00. 00010000:00000010:1.1:1775773840.916193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c1ec600. 00010000:00000010:1.1:1775773840.916196:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d37100. 00010000:00000010:1.1:1775773840.916199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206179a00. 00010000:00000010:1.1:1775773840.916202:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d37600. 00010000:00000010:1.1:1775773840.916205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206179000. 00010000:00000010:1.1:1775773840.916207:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb7900. 00010000:00000010:1.1:1775773840.916210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202d35200. 00010000:00000010:1.1:1775773840.916213:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb7700. 00010000:00000010:1.1:1775773840.916216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202d34800. 00010000:00000010:1.1:1775773840.916220:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234addf00. 00010000:00000010:1.1:1775773840.916222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da43e00. 00010000:00000010:1.1:1775773840.916226:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618c200. 00010000:00000010:1.1:1775773840.916229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202d34a00. 00010000:00000010:1.1:1775773840.916232:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234add000. 00010000:00000010:1.1:1775773840.916234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da43200. 00010000:00000010:1.1:1775773840.916237:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d37f00. 00010000:00000010:1.1:1775773840.916240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206179e00. 00010000:00000010:1.1:1775773840.916243:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d37000. 00010000:00000010:1.1:1775773840.916245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206179200. 00010000:00000010:1.1:1775773840.916248:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d37500. 00010000:00000010:1.1:1775773840.916251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206178400. 00010000:00000010:1.1:1775773840.916254:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d37b00. 00010000:00000010:1.1:1775773840.916256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206178800. 00010000:00000010:1.1:1775773840.916259:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d37c00. 00010000:00000010:1.1:1775773840.916263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226383800. 00010000:00000010:1.1:1775773840.916266:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d37300. 00010000:00000010:1.1:1775773840.916270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226382600. 00010000:00000010:1.1:1775773840.916273:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205603d00. 00010000:00000010:1.1:1775773840.916276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226382000. 00010000:00000010:1.1:1775773840.916279:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205603400. 00010000:00000010:1.1:1775773840.916282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226382a00. 00010000:00000010:1.1:1775773840.916285:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205603800. 00010000:00000010:1.1:1775773840.916288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226383400. 00010000:00000010:1.1:1775773840.916291:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205603e00. 00010000:00000010:1.1:1775773840.916294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226383c00. 00010000:00000010:1.1:1775773840.916297:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cca4400. 00010000:00000010:1.1:1775773840.916300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202d34c00. 00010000:00000010:1.1:1775773840.916303:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205603900. 00010000:00000010:1.1:1775773840.916306:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c1ed200. 00010000:00000010:1.1:1775773840.916309:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cca4800. 00010000:00000010:1.1:1775773840.916312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c1ec800. 00010000:00000010:1.1:1775773840.916315:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205603700. 00010000:00000010:1.1:1775773840.916318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226382c00. 00010000:00000010:1.1:1775773840.916320:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234add500. 00010000:00000010:1.1:1775773840.916323:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da42400. 00010000:00000010:1.1:1775773840.916326:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cca4e00. 00010000:00000010:1.1:1775773840.916328:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226383600. 00010000:00000010:1.1:1775773840.916331:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205603a00. 00010000:00000010:1.1:1775773840.916334:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226382e00. 00010000:00000010:1.1:1775773840.916336:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205603100. 00010000:00000010:1.1:1775773840.916339:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226382200. 00010000:00000010:1.1:1775773840.916342:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205603600. 00010000:00000010:1.1:1775773840.916345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226383a00. 00010000:00000010:1.1:1775773840.916348:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205603f00. 00010000:00000010:1.1:1775773840.916351:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226383000. 00010000:00000010:1.1:1775773840.916354:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618cd00. 00010000:00000010:1.1:1775773840.916357:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202d35800. 00010000:00000010:1.1:1775773840.916359:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234addb00. 00010000:00000010:1.1:1775773840.916362:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c1ed000. 00010000:00000010:1.1:1775773840.916365:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618c400. 00010000:00000010:1.1:1775773840.916368:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202d34400. 00010000:00000010:1.1:1775773840.916370:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cca4900. 00010000:00000010:1.1:1775773840.916373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202d34200. 00010000:00000010:1.1:1775773840.916375:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234addc00. 00010000:00000010:1.1:1775773840.916378:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da42800. 00010000:00000010:1.1:1775773840.916381:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234add300. 00010000:00000010:1.1:1775773840.916384:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da43800. 00010000:00000010:1.1:1775773840.916387:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234add200. 00010000:00000010:1.1:1775773840.916389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da42600. 00010000:00000010:1.1:1775773840.916392:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205603000. 00010000:00000010:1.1:1775773840.916395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202d34e00. 00010000:00000010:1.1:1775773840.916398:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205603500. 00010000:00000010:1.1:1775773840.916401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226383e00. 00010000:00000010:1.1:1775773840.916404:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205603b00. 00010000:00000010:1.1:1775773840.916406:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226383200. 00010000:00000010:1.1:1775773840.916409:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205603c00. 00010000:00000010:1.1:1775773840.916412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202d35a00. 00010000:00000010:1.1:1775773840.916414:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205603300. 00010000:00000010:1.1:1775773840.916417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226382400. 00010000:00000010:1.1:1775773840.916420:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618c800. 00010000:00000010:1.1:1775773840.916423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c1ec400. 00010000:00000010:1.1:1775773840.916425:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205603200. 00010000:00000010:1.1:1775773840.916428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226382800. 00010000:00000010:1.1:1775773840.916431:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb6600. 00010000:00000010:1.1:1775773840.916435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de68c00. 00010000:00000010:1.1:1775773840.916438:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb6500. 00010000:00000010:1.1:1775773840.916440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de68200. 00010000:00000010:1.1:1775773840.916443:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb6b00. 00010000:00000010:1.1:1775773840.916446:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de69a00. 00010000:00000010:1.1:1775773840.916449:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234addd00. 00010000:00000010:1.1:1775773840.916452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c1ed400. 00010000:00000010:1.1:1775773840.916454:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb6c00. 00010000:00000010:1.1:1775773840.916457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de69000. 00010000:00000010:1.1:1775773840.916460:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb6300. 00010000:00000010:1.1:1775773840.916463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de69e00. 00010000:00000010:1.1:1775773840.916466:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb6200. 00010000:00000010:1.1:1775773840.916469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de69200. 00010000:00000010:1.1:1775773840.916472:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb6d00. 00010000:00000010:1.1:1775773840.916474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de68400. 00010000:00000010:1.1:1775773840.916477:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb6400. 00010000:00000010:1.1:1775773840.916480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de68800. 00010000:00000010:1.1:1775773840.916483:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb6800. 00010000:00000010:1.1:1775773840.916486:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de69800. 00010000:00000010:1.1:1775773840.916489:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb6e00. 00010000:00000010:1.1:1775773840.916491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de68600. 00010000:00000010:1.1:1775773840.916494:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb6900. 00010000:00000010:1.1:1775773840.916497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623472ec00. 00010000:00000010:1.1:1775773840.916501:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234add400. 00010000:00000010:1.1:1775773840.916503:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5c9e00. 00010000:00000010:1.1:1775773840.916506:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb6700. 00010000:00000010:1.1:1775773840.916509:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de68000. 00010000:00000010:1.1:1775773840.916512:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207947100. 00010000:00000010:1.1:1775773840.916515:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623472f400. 00010000:00000010:1.1:1775773840.916518:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234add800. 00010000:00000010:1.1:1775773840.916520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de68a00. 00010000:00000010:1.1:1775773840.916523:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb6a00. 00010000:00000010:1.1:1775773840.916526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de69400. 00010000:00000010:1.1:1775773840.916529:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb6100. 00010000:00000010:1.1:1775773840.916532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de69c00. 00010000:00000010:1.1:1775773840.916534:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618ce00. 00010000:00000010:1.1:1775773840.916537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623472e200. 00010000:00000010:1.1:1775773840.916540:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234adde00. 00010000:00000010:1.1:1775773840.916542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de69600. 00010000:00000010:1.1:1775773840.916545:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb6f00. 00010000:00000010:1.1:1775773840.916548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621de68e00. 00010000:00000010:1.1:1775773840.916550:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb6000. 00010000:00000010:1.1:1775773840.916553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c656a00. 00010000:00000010:1.1:1775773840.916557:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dd3cf00. 00010000:00000010:1.1:1775773840.916560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c657400. 00010000:00000010:1.1:1775773840.916563:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dd3c000. 00010000:00000010:1.1:1775773840.916566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c657c00. 00010000:00000010:1.1:1775773840.916570:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211989100. 00010000:00000010:1.1:1775773840.916573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5c9200. 00010000:00000010:1.1:1775773840.916575:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211989600. 00010000:00000010:1.1:1775773840.916578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c656c00. 00010000:00000010:1.1:1775773840.916581:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618c900. 00010000:00000010:1.1:1775773840.916584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623472f000. 00010000:00000010:1.1:1775773840.916587:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618c700. 00010000:00000010:1.1:1775773840.916590:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623472fa00. 00010000:00000010:1.1:1775773840.916592:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dd3c500. 00010000:00000010:1.1:1775773840.916595:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c657600. 00010000:00000010:1.1:1775773840.916598:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dd3cb00. 00010000:00000010:1.1:1775773840.916601:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c656e00. 00010000:00000010:1.1:1775773840.916604:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211989f00. 00010000:00000010:1.1:1775773840.916607:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c656200. 00010000:00000010:1.1:1775773840.916609:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211989000. 00010000:00000010:1.1:1775773840.916612:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c657a00. 00010000:00000010:1.1:1775773840.916615:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dd3cc00. 00010000:00000010:1.1:1775773840.916618:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623472e400. 00010000:00000010:1.1:1775773840.916621:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211989500. 00010000:00000010:1.1:1775773840.916624:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c657000. 00010000:00000010:1.1:1775773840.916627:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dd3c300. 00010000:00000010:1.1:1775773840.916629:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c657e00. 00010000:00000010:1.1:1775773840.916632:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211989b00. 00010000:00000010:1.1:1775773840.916635:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c657200. 00010000:00000010:1.1:1775773840.916637:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618ca00. 00010000:00000010:1.1:1775773840.916640:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623472e000. 00010000:00000010:1.1:1775773840.916643:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618c100. 00010000:00000010:1.1:1775773840.916646:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623472fc00. 00010000:00000010:1.1:1775773840.916648:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dd3c200. 00010000:00000010:1.1:1775773840.916651:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c656400. 00010000:00000010:1.1:1775773840.916654:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dd3cd00. 00010000:00000010:1.1:1775773840.916656:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c656800. 00010000:00000010:1.1:1775773840.916659:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dd3c400. 00010000:00000010:1.1:1775773840.916662:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c657800. 00010000:00000010:1.1:1775773840.916665:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dd3c800. 00010000:00000010:1.1:1775773840.916668:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c656600. 00010000:00000010:1.1:1775773840.916671:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dd3ce00. 00010000:00000010:1.1:1775773840.916673:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c656000. 00010000:00000010:1.1:1775773840.916675:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dd3c900. 00010000:00000010:1.1:1775773840.916679:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233923000. 00010000:00000010:1.1:1775773840.916682:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211989c00. 00010000:00000010:1.1:1775773840.916684:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623472f600. 00010000:00000010:1.1:1775773840.916687:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618c600. 00010000:00000010:1.1:1775773840.916689:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623472e600. 00010000:00000010:1.1:1775773840.916692:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207947400. 00010000:00000010:1.1:1775773840.916695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623472f800. 00010000:00000010:1.1:1775773840.916697:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dd3c700. 00010000:00000010:1.1:1775773840.916700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233923e00. 00010000:00000010:1.1:1775773840.916703:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211989300. 00010000:00000010:1.1:1775773840.916706:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623472e800. 00010000:00000010:1.1:1775773840.916709:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dd3ca00. 00010000:00000010:1.1:1775773840.916712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233923200. 00010000:00000010:1.1:1775773840.916715:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dd3c100. 00010000:00000010:1.1:1775773840.916718:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233922400. 00010000:00000010:1.1:1775773840.916721:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211989200. 00010000:00000010:1.1:1775773840.916724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233922800. 00010000:00000010:1.1:1775773840.916726:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621dd3c600. 00010000:00000010:1.1:1775773840.916729:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202d34000. 00010000:00000010:1.1:1775773840.916732:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211989d00. 00010000:00000010:1.1:1775773840.916735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233923800. 00010000:00000010:1.1:1775773840.916738:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205761400. 00010000:00000010:1.1:1775773840.916742:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233922600. 00010000:00000010:1.1:1775773840.916745:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211989400. 00010000:00000010:1.1:1775773840.916747:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623472fe00. 00010000:00000010:1.1:1775773840.916751:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207947800. 00010000:00000010:1.1:1775773840.916754:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202d35600. 00010000:00000010:1.1:1775773840.916756:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211989800. 00010000:00000010:1.1:1775773840.916759:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5c8400. 00010000:00000010:1.1:1775773840.916762:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205761800. 00010000:00000010:1.1:1775773840.916765:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202d34600. 00010000:00000010:1.1:1775773840.916768:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211989e00. 00010000:00000010:1.1:1775773840.916770:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623472f200. 00010000:00000010:1.1:1775773840.916773:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205761e00. 00010000:00000010:1.1:1775773840.916776:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233922000. 00010000:00000010:1.1:1775773840.916779:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618cf00. 00010000:00000010:1.1:1775773840.916781:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623472ee00. 00010000:00000010:1.1:1775773840.916784:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205761900. 00010000:00000010:1.1:1775773840.916786:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233922a00. 00010000:00000010:1.1:1775773840.916789:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205761700. 00010000:00000010:1.1:1775773840.916792:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233923400. 00010000:00000010:1.1:1775773840.916795:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211989900. 00010000:00000010:1.1:1775773840.916798:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233923c00. 00010000:00000010:1.1:1775773840.916800:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211989700. 00010000:00000010:1.1:1775773840.916803:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233922c00. 00010000:00000010:1.1:1775773840.916806:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205761a00. 00010000:00000010:1.1:1775773840.916808:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233923600. 00010000:00000010:1.1:1775773840.916811:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618c000. 00010000:00000010:1.1:1775773840.916814:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623472ea00. 00010000:00000010:1.1:1775773840.916816:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205761100. 00010000:00000010:1.1:1775773840.916819:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233922e00. 00010000:00000010:1.1:1775773840.916821:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205761600. 00010000:00000010:1.1:1775773840.916824:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233922200. 00010000:00000010:1.1:1775773840.916840:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205761f00. 00010000:00000010:1.1:1775773840.916844:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233923a00. 00010000:00000010:1.1:1775773840.916847:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205761000. 00010000:00000010:1.1:1775773840.916850:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcf400. 00010000:00000010:1.1:1775773840.916853:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205761500. 00010000:00000010:1.1:1775773840.916856:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcfc00. 00010000:00000010:1.1:1775773840.916859:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6211989a00. 00010000:00000010:1.1:1775773840.916862:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcec00. 00010000:00000010:1.1:1775773840.916865:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205761b00. 00010000:00000010:1.1:1775773840.916868:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc4be00. 00010000:00000010:1.1:1775773840.916871:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207947e00. 00010000:00000010:1.1:1775773840.916874:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202d35000. 00010000:00000010:1.1:1775773840.916877:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202fc3a00. 00010000:00000010:1.1:1775773840.916879:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcf600. 00010000:00000010:1.1:1775773840.916882:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205761c00. 00010000:00000010:1.1:1775773840.916885:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcee00. 00010000:00000010:1.1:1775773840.916887:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618c500. 00010000:00000010:1.1:1775773840.916890:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc4b600. 00010000:00000010:1.1:1775773840.916893:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205761300. 00010000:00000010:1.1:1775773840.916896:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bce200. 00010000:00000010:1.1:1775773840.916898:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202fc3100. 00010000:00000010:1.1:1775773840.916901:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcfa00. 00010000:00000010:1.1:1775773840.916904:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205761200. 00010000:00000010:1.1:1775773840.916906:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcf000. 00010000:00000010:1.1:1775773840.916909:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618cb00. 00010000:00000010:1.1:1775773840.916912:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc4a000. 00010000:00000010:1.1:1775773840.916915:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618cc00. 00010000:00000010:1.1:1775773840.916917:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc4b000. 00010000:00000010:1.1:1775773840.916919:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6205761d00. 00010000:00000010:1.1:1775773840.916922:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcfe00. 00010000:00000010:1.1:1775773840.916925:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202fc3600. 00010000:00000010:1.1:1775773840.916927:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcf200. 00010000:00000010:1.1:1775773840.916931:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233731000. 00010000:00000010:1.1:1775773840.916967:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bce400. 00010000:00000010:1.1:1775773840.916972:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233731500. 00010000:00000010:1.1:1775773840.916974:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bce800. 00010000:00000010:1.1:1775773840.916978:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618c300. 00010000:00000010:1.1:1775773840.916984:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc4bc00. 00010000:00000010:1.1:1775773840.916988:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623491e700. 00010000:00000010:1.1:1775773840.916990:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcf800. 00010000:00000010:1.1:1775773840.916994:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233731b00. 00010000:00000010:1.1:1775773840.916997:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bce600. 00010000:00000010:1.1:1775773840.916999:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233731c00. 00010000:00000010:1.1:1775773840.917002:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bce000. 00010000:00000010:1.1:1775773840.917004:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233731300. 00010000:00000010:1.1:1775773840.917006:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233bcea00. 00010000:00000010:1.1:1775773840.917009:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233731200. 00010000:00000010:1.1:1775773840.917012:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b48c00. 00010000:00000010:1.1:1775773840.917015:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233731d00. 00010000:00000010:1.1:1775773840.917017:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc4ac00. 00010000:00000010:1.1:1775773840.917020:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6207947900. 00010000:00000010:1.1:1775773840.917026:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202d35400. 00010000:00000010:1.1:1775773840.917030:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233731400. 00010000:00000010:1.1:1775773840.917032:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6202d35c00. 00010000:00000010:1.1:1775773840.917035:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202fc3f00. 00010000:00000010:1.1:1775773840.917038:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b48200. 00010000:00000010:1.1:1775773840.917041:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623491ea00. 00010000:00000010:1.1:1775773840.917043:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc4b400. 00010000:00000010:1.1:1775773840.917046:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233731800. 00010000:00000010:1.1:1775773840.917049:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b49a00. 00010000:00000010:1.1:1775773840.917052:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202fc3000. 00010000:00000010:1.1:1775773840.917055:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62339f0c00. 00010000:00000010:1.1:1775773840.917058:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233731e00. 00010000:00000010:1.1:1775773840.917060:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62339f1400. 00010000:00000010:1.1:1775773840.917063:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233731900. 00010000:00000010:1.1:1775773840.917066:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b49000. 00010000:00000010:1.1:1775773840.917069:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623491e100. 00010000:00000010:1.1:1775773840.917071:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62339f1600. 00010000:00000010:1.1:1775773840.917082:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233731700. 00010000:00000010:1.1:1775773840.917085:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b49e00. 00010000:00000010:1.1:1775773840.917088:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233731a00. 00010000:00000010:1.1:1775773840.917091:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b49200. 00010000:00000010:1.1:1775773840.917094:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623491e600. 00010000:00000010:1.1:1775773840.917097:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b48400. 00010000:00000010:1.1:1775773840.917099:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233731100. 00010000:00000010:1.1:1775773840.917102:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b48800. 00010000:00000010:1.1:1775773840.917104:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233731600. 00010000:00000010:1.1:1775773840.917107:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b49800. 00010000:00000010:1.1:1775773840.917109:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202fc3500. 00010000:00000010:1.1:1775773840.917111:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b48600. 00010000:00000010:1.1:1775773840.917114:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233731f00. 00010000:00000010:1.1:1775773840.917116:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b48000. 00010000:00000010:1.1:1775773840.917119:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f623491ef00. 00010000:00000010:1.1:1775773840.917125:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc4a800. 00010000:00000010:1.1:1775773840.917146:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62346e6900. 00010000:00000010:1.1:1775773840.917150:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b48a00. 00010000:00000010:1.1:1775773840.917153:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202fc3b00. 00010000:00000010:1.1:1775773840.917160:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5c8800. 00010000:00000010:1.1:1775773840.917163:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62346e6800. 00010000:00000010:1.1:1775773840.917167:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b49400. 00010000:00000010:1.1:1775773840.917169:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62346e6e00. 00010000:00000010:1.1:1775773840.917175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6211b49c00. 00000800:00000200:1.1:1775773845.819827:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773845.819831:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775773845.820277:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773845.820279:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775773845.820560:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773845.820562:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775773850.939911:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773850.940195:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773850.940371:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000080:00000010:2.1:1775773851.074152:0:207610:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6206644be0. 00000080:00000010:2.1:1775773851.102166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7ee0. 00000080:00000010:2.1:1775773851.102175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7ae0. 00000080:00000010:2.1:1775773851.102179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df700c0. 00000080:00000010:2.1:1775773851.102183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7560. 00000080:00000010:2.1:1775773851.102186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c71a0. 00000080:00000010:2.1:1775773851.102189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621de656e0. 00000080:00000010:2.1:1775773851.102193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25d00. 00000080:00000010:2.1:1775773851.102197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7020. 00000080:00000010:2.1:1775773851.102200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70a20. 00000080:00000010:2.1:1775773851.102202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70f40. 00000080:00000010:2.1:1775773851.102205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70fe0. 00000080:00000010:2.1:1775773851.102207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25a60. 00000080:00000010:2.1:1775773851.102211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c71c0. 00000080:00000010:2.1:1775773851.102214:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c71e0. 00000080:00000010:2.1:1775773851.102217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7540. 00000080:00000010:2.1:1775773851.102228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7480. 00000080:00000010:2.1:1775773851.102231:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7e40. 00000080:00000010:2.1:1775773851.102234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7e80. 00000080:00000010:2.1:1775773851.106160:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70740. 00000080:00000010:2.1:1775773851.106167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c254c0. 00000080:00000010:2.1:1775773851.106171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25ec0. 00000080:00000010:2.1:1775773851.106175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7060. 00000080:00000010:2.1:1775773851.106179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c70a0. 00000080:00000010:2.1:1775773851.106182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7fc0. 00000080:00000010:2.1:1775773851.106184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7b40. 00000080:00000010:2.1:1775773851.106187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70120. 00000080:00000010:2.1:1775773851.106190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df701c0. 00000080:00000010:2.1:1775773851.106192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7300. 00000080:00000010:2.1:1775773851.106195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a15420. 00000080:00000010:2.1:1775773851.106198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203674e0. 00000080:00000010:2.1:1775773851.106201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207a158a0. 00000080:00000010:2.1:1775773851.106203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70e60. 00000080:00000010:2.1:1775773851.106206:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7840. 00000080:00000010:2.1:1775773851.106209:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7620. 00000080:00000010:2.1:1775773851.106212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7f80. 00000080:00000010:2.1:1775773851.106214:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7600. 00000080:00000010:2.1:1775773851.106219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7320. 00000080:00000010:2.1:1775773851.106222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70380. 00000080:00000010:2.1:1775773851.106225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25380. 00000080:00000010:2.1:1775773851.106227:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70520. 00000080:00000010:2.1:1775773851.106231:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70d40. 00000080:00000010:2.1:1775773851.106234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62203fa420. 00000080:00000010:2.1:1775773851.106237:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c76c0. 00000080:00000010:2.1:1775773851.106240:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70360. 00000080:00000010:2.1:1775773851.106242:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f620572f3e0. 00000080:00000010:2.1:1775773851.106245:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7c00. 00000080:00000010:2.1:1775773851.106247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7fa0. 00000080:00000010:2.1:1775773851.106250:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25f40. 00000080:00000010:2.1:1775773851.106253:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25cc0. 00000080:00000010:2.1:1775773851.106257:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7de0. 00000080:00000010:2.1:1775773851.106262:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06c3e0. 00000080:00000010:2.1:1775773851.106266:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7720. 00000080:00000010:2.1:1775773851.106270:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7780. 00000080:00000010:2.1:1775773851.106274:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7180. 00000080:00000010:2.1:1775773851.106280:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7360. 00000080:00000010:2.1:1775773851.106285:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7240. 00000080:00000010:2.1:1775773851.106290:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7700. 00000080:00000010:2.1:1775773851.106295:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7aa0. 00000080:00000010:2.1:1775773851.106299:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c74c0. 00000080:00000010:2.1:1775773851.106303:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7820. 00000080:00000010:2.1:1775773851.106307:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c258e0. 00000080:00000010:2.1:1775773851.106312:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a88e0. 00000080:00000010:2.1:1775773851.106316:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7260. 00000080:00000010:2.1:1775773851.106319:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7d00. 00000080:00000010:2.1:1775773851.106321:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25c20. 00000080:00000010:2.1:1775773851.106324:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c256a0. 00000080:00000010:2.1:1775773851.106327:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70700. 00000080:00000010:2.1:1775773851.106329:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25800. 00000080:00000010:2.1:1775773851.106332:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70260. 00000080:00000010:2.1:1775773851.106334:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8980. 00000080:00000010:2.1:1775773851.106337:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a89c0. 00000080:00000010:2.1:1775773851.106339:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c77c0. 00000080:00000010:2.1:1775773851.106341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7140. 00000080:00000010:2.1:1775773851.106344:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7d40. 00000080:00000010:2.1:1775773851.106346:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7a60. 00000080:00000010:2.1:1775773851.106348:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7340. 00000080:00000010:2.1:1775773851.106351:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c250c0. 00000080:00000010:2.1:1775773851.106353:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7900. 00000080:00000010:2.1:1775773851.106356:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7380. 00000080:00000010:2.1:1775773851.106359:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7760. 00000080:00000010:2.1:1775773851.106361:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70020. 00000080:00000010:2.1:1775773851.106363:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8bc0. 00000080:00000010:2.1:1775773851.106365:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70a60. 00000080:00000010:2.1:1775773851.106367:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25f20. 00000080:00000010:2.1:1775773851.106369:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7800. 00000080:00000010:2.1:1775773851.106372:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7c40. 00000080:00000010:2.1:1775773851.106374:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c76a0. 00000080:00000010:2.1:1775773851.106377:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c70c0. 00000080:00000010:2.1:1775773851.106379:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7e20. 00000080:00000010:2.1:1775773851.106381:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7460. 00000080:00000010:2.1:1775773851.106383:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c79e0. 00000080:00000010:2.1:1775773851.106391:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70440. 00000080:00000010:2.1:1775773851.106394:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70200. 00000080:00000010:2.1:1775773851.106396:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25120. 00000080:00000010:2.1:1775773851.106399:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7d20. 00000080:00000010:2.1:1775773851.106402:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8d60. 00000080:00000010:2.1:1775773851.106406:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7b00. 00000080:00000010:2.1:1775773851.106409:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c75a0. 00000080:00000010:2.1:1775773851.106412:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7500. 00000080:00000010:2.1:1775773851.106415:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a86e0. 00000080:00000010:2.1:1775773851.106419:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8140. 00000080:00000010:2.1:1775773851.106422:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25ca0. 00000080:00000010:2.1:1775773851.106426:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a8fa0. 00000080:00000010:2.1:1775773851.106429:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7dc0. 00000080:00000010:2.1:1775773851.106433:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62339a82c0. 00000080:00000010:2.1:1775773851.106436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c253e0. 00000080:00000010:2.1:1775773851.106440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62060ec400. 00010000:00000010:2.1:1775773851.106446:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622d964400. 00010000:00000010:2.1:1775773851.106451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621839e000. 00010000:00000010:2.1:1775773851.106455:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233960100. 00010000:00000010:2.1:1775773851.106458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226128a00. 00010000:00000010:2.1:1775773851.106460:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233960600. 00010000:00000010:2.1:1775773851.106462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226129400. 00010000:00000010:2.1:1775773851.106465:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234adce00. 00010000:00000010:2.1:1775773851.106467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d7600. 00010000:00000010:2.1:1775773851.106469:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233960f00. 00010000:00000010:2.1:1775773851.106471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226129c00. 00010000:00000010:2.1:1775773851.106473:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233960000. 00010000:00000010:2.1:1775773851.106474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226128c00. 00010000:00000010:2.1:1775773851.106476:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622d964800. 00010000:00000010:2.1:1775773851.106478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621839ea00. 00010000:00000010:2.1:1775773851.106480:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234adc900. 00010000:00000010:2.1:1775773851.106481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623476c000. 00010000:00000010:2.1:1775773851.106525:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233960500. 00010000:00000010:2.1:1775773851.106527:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226129600. 00010000:00000010:2.1:1775773851.106529:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234adc700. 00010000:00000010:2.1:1775773851.106532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d6a00. 00010000:00000010:2.1:1775773851.106534:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234adca00. 00010000:00000010:2.1:1775773851.106535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d7800. 00010000:00000010:2.1:1775773851.106538:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622d964e00. 00010000:00000010:2.1:1775773851.106540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d7e00. 00010000:00000010:2.1:1775773851.106542:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e5200. 00010000:00000010:2.1:1775773851.106569:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621e5c9000. 00010000:00000010:2.1:1775773851.106575:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622d964900. 00010000:00000010:2.1:1775773851.106577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cf8e00. 00010000:00000010:2.1:1775773851.106579:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e5d00. 00010000:00000010:2.1:1775773851.106581:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cf8200. 00010000:00000010:2.1:1775773851.106583:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e5400. 00010000:00000010:2.1:1775773851.106585:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cf9a00. 00010000:00000010:2.1:1775773851.106587:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e5800. 00010000:00000010:2.1:1775773851.106588:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cf9000. 00010000:00000010:2.1:1775773851.106590:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e5e00. 00010000:00000010:2.1:1775773851.106591:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cf9e00. 00010000:00000010:2.1:1775773851.106593:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e5900. 00010000:00000010:2.1:1775773851.106595:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cf9200. 00010000:00000010:2.1:1775773851.106596:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e5700. 00010000:00000010:2.1:1775773851.106598:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d6800. 00010000:00000010:2.1:1775773851.106600:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234adc100. 00010000:00000010:2.1:1775773851.106601:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225845e00. 00010000:00000010:2.1:1775773851.106603:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234adc600. 00010000:00000010:2.1:1775773851.106605:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225844200. 00010000:00000010:2.1:1775773851.106607:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cca4700. 00010000:00000010:2.1:1775773851.106609:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cf8400. 00010000:00000010:2.1:1775773851.106610:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e5a00. 00010000:00000010:2.1:1775773851.106612:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cf8800. 00010000:00000010:2.1:1775773851.106613:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e5100. 00010000:00000010:2.1:1775773851.106615:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cf9800. 00010000:00000010:2.1:1775773851.106616:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e5600. 00010000:00000010:2.1:1775773851.106619:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cf8600. 00010000:00000010:2.1:1775773851.106620:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb7100. 00010000:00000010:2.1:1775773851.106622:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d6000. 00010000:00000010:2.1:1775773851.106624:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e5f00. 00010000:00000010:2.1:1775773851.106626:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d7400. 00010000:00000010:2.1:1775773851.106628:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e5000. 00010000:00000010:2.1:1775773851.106629:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cf8000. 00010000:00000010:2.1:1775773851.106631:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622d964700. 00010000:00000010:2.1:1775773851.106633:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621839f400. 00010000:00000010:2.1:1775773851.106634:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622d964a00. 00010000:00000010:2.1:1775773851.106636:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621839fc00. 00010000:00000010:2.1:1775773851.106637:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cca4a00. 00010000:00000010:2.1:1775773851.106639:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621839ec00. 00010000:00000010:2.1:1775773851.106641:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622d964100. 00010000:00000010:2.1:1775773851.106642:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d6c00. 00010000:00000010:2.1:1775773851.106644:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622d964600. 00010000:00000010:2.1:1775773851.106645:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cf8a00. 00010000:00000010:2.1:1775773851.106647:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622d964f00. 00010000:00000010:2.1:1775773851.106667:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cf9400. 00010000:00000010:2.1:1775773851.106670:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e5500. 00010000:00000010:2.1:1775773851.106671:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cf9c00. 00010000:00000010:2.1:1775773851.106673:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb7600. 00010000:00000010:2.1:1775773851.106674:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cf8c00. 00010000:00000010:2.1:1775773851.106676:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e5b00. 00010000:00000010:2.1:1775773851.106677:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6207cf9600. 00010000:00000010:2.1:1775773851.106679:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb7f00. 00010000:00000010:2.1:1775773851.106681:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d6600. 00010000:00000010:2.1:1775773851.106682:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb7000. 00010000:00000010:2.1:1775773851.106684:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225845000. 00010000:00000010:2.1:1775773851.106686:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb7500. 00010000:00000010:2.1:1775773851.106687:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d7200. 00010000:00000010:2.1:1775773851.106689:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb7b00. 00010000:00000010:2.1:1775773851.106691:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225845a00. 00010000:00000010:2.1:1775773851.106693:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622d964000. 00010000:00000010:2.1:1775773851.106695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621839f600. 00010000:00000010:2.1:1775773851.106697:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622d964500. 00010000:00000010:2.1:1775773851.106699:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621198a800. 00010000:00000010:2.1:1775773851.106701:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622d964b00. 00010000:00000010:2.1:1775773851.106703:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d7c00. 00010000:00000010:2.1:1775773851.106705:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622d964c00. 00010000:00000010:2.1:1775773851.106706:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621839ee00. 00010000:00000010:2.1:1775773851.106708:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622d964300. 00010000:00000010:2.1:1775773851.106710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621198b800. 00010000:00000010:2.1:1775773851.106711:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e5c00. 00010000:00000010:2.1:1775773851.106713:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621198a600. 00010000:00000010:2.1:1775773851.106715:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62259e5300. 00010000:00000010:2.1:1775773851.106716:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225845600. 00010000:00000010:2.1:1775773851.106719:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621029e400. 00010000:00000010:2.1:1775773851.106721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225844e00. 00010000:00000010:2.1:1775773851.106723:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621029e800. 00010000:00000010:2.1:1775773851.106724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621198a000. 00010000:00000010:2.1:1775773851.106727:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de6bf00. 00010000:00000010:2.1:1775773851.106729:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621839e200. 00010000:00000010:2.1:1775773851.106730:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de6bd00. 00010000:00000010:2.1:1775773851.106732:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621198aa00. 00010000:00000010:2.1:1775773851.106734:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de6b400. 00010000:00000010:2.1:1775773851.106735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621198b400. 00010000:00000010:2.1:1775773851.106737:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621029ee00. 00010000:00000010:2.1:1775773851.106739:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621198bc00. 00010000:00000010:2.1:1775773851.106740:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621029e900. 00010000:00000010:2.1:1775773851.106742:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621198ac00. 00010000:00000010:2.1:1775773851.106743:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621029e700. 00010000:00000010:2.1:1775773851.106745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621198b600. 00010000:00000010:2.1:1775773851.106746:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621029ea00. 00010000:00000010:2.1:1775773851.106749:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621198ae00. 00010000:00000010:2.1:1775773851.106750:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621029e100. 00010000:00000010:2.1:1775773851.106752:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621198a200. 00010000:00000010:2.1:1775773851.106754:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621029e600. 00010000:00000010:2.1:1775773851.106756:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621198ba00. 00010000:00000010:2.1:1775773851.106757:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621029ef00. 00010000:00000010:2.1:1775773851.106759:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621198b000. 00010000:00000010:2.1:1775773851.106761:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cca4100. 00010000:00000010:2.1:1775773851.106762:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225845800. 00010000:00000010:2.1:1775773851.106764:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de6b800. 00010000:00000010:2.1:1775773851.106766:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621839fa00. 00010000:00000010:2.1:1775773851.106767:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621029e000. 00010000:00000010:2.1:1775773851.106769:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621198be00. 00010000:00000010:2.1:1775773851.106770:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621029e500. 00010000:00000010:2.1:1775773851.106772:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621198b200. 00010000:00000010:2.1:1775773851.106773:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cca4600. 00010000:00000010:2.1:1775773851.106775:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225844c00. 00010000:00000010:2.1:1775773851.106777:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cca4f00. 00010000:00000010:2.1:1775773851.106778:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225845c00. 00010000:00000010:2.1:1775773851.106780:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de6be00. 00010000:00000010:2.1:1775773851.106782:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c1eda00. 00010000:00000010:2.1:1775773851.106784:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621029eb00. 00010000:00000010:2.1:1775773851.106785:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225845400. 00010000:00000010:2.1:1775773851.106787:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb7c00. 00010000:00000010:2.1:1775773851.106788:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c1ed600. 00010000:00000010:2.1:1775773851.106790:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de6b900. 00010000:00000010:2.1:1775773851.106791:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621839f000. 00010000:00000010:2.1:1775773851.106793:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de6b700. 00010000:00000010:2.1:1775773851.106795:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621839fe00. 00010000:00000010:2.1:1775773851.106796:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621029ec00. 00010000:00000010:2.1:1775773851.106798:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621198a400. 00010000:00000010:2.1:1775773851.106799:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621029e300. 00010000:00000010:2.1:1775773851.106802:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62258c8a00. 00010000:00000010:2.1:1775773851.106803:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621029e200. 00010000:00000010:2.1:1775773851.106805:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62258c9c00. 00010000:00000010:2.1:1775773851.106807:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621029ed00. 00010000:00000010:2.1:1775773851.106809:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62258c8c00. 00010000:00000010:2.1:1775773851.106811:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62258cb300. 00010000:00000010:2.1:1775773851.106813:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62258c9600. 00010000:00000010:2.1:1775773851.106815:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62258cb200. 00010000:00000010:2.1:1775773851.106816:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225844600. 00010000:00000010:2.1:1775773851.106818:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62258cbd00. 00010000:00000010:2.1:1775773851.106820:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62258c8e00. 00010000:00000010:2.1:1775773851.106821:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cca4000. 00010000:00000010:2.1:1775773851.106823:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62258c8200. 00010000:00000010:2.1:1775773851.106824:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62258cb400. 00010000:00000010:2.1:1775773851.106826:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62258c9a00. 00010000:00000010:2.1:1775773851.106827:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62258cb800. 00010000:00000010:2.1:1775773851.106829:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c1ed800. 00010000:00000010:2.1:1775773851.106830:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de6ba00. 00010000:00000010:2.1:1775773851.106832:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621839f200. 00010000:00000010:2.1:1775773851.106834:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62258cbe00. 00010000:00000010:2.1:1775773851.106835:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c1ede00. 00010000:00000010:2.1:1775773851.106837:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cca4500. 00010000:00000010:2.1:1775773851.106838:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225844000. 00010000:00000010:2.1:1775773851.106840:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62258cb900. 00010000:00000010:2.1:1775773851.106842:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62258c9000. 00010000:00000010:2.1:1775773851.106843:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62258cb700. 00010000:00000010:2.1:1775773851.106845:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62258c9e00. 00010000:00000010:2.1:1775773851.106846:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62258cba00. 00010000:00000010:2.1:1775773851.106848:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62258c9200. 00010000:00000010:2.1:1775773851.106850:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62258cb100. 00010000:00000010:2.1:1775773851.106851:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62258c8400. 00010000:00000010:2.1:1775773851.106853:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62258cb600. 00010000:00000010:2.1:1775773851.106855:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62258c8800. 00010000:00000010:2.1:1775773851.106857:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62258cbf00. 00010000:00000010:2.1:1775773851.106858:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62258c9800. 00010000:00000010:2.1:1775773851.106860:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cca4b00. 00010000:00000010:2.1:1775773851.106862:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62258c8600. 00010000:00000010:2.1:1775773851.106864:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cca4c00. 00010000:00000010:2.1:1775773851.106865:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c1eca00. 00010000:00000010:2.1:1775773851.106867:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62258cb000. 00010000:00000010:2.1:1775773851.106868:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c1ecc00. 00010000:00000010:2.1:1775773851.106870:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622cca4300. 00010000:00000010:2.1:1775773851.106875:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225844800. 00010000:00000010:2.1:1775773851.106877:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62258cb500. 00010000:00000010:2.1:1775773851.106879:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62258c8000. 00010000:00000010:2.1:1775773851.106880:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de6b100. 00010000:00000010:2.1:1775773851.106882:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621839e400. 00010000:00000010:2.1:1775773851.106884:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de6b600. 00010000:00000010:2.1:1775773851.106885:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62258c9400. 00010000:00000010:2.1:1775773851.106887:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62258cbb00. 00010000:00000010:2.1:1775773851.106888:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206207a00. 00010000:00000010:2.1:1775773851.106890:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f62258cbc00. 00010000:00000010:2.1:1775773851.106892:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206207000. 00010000:00000010:2.1:1775773851.106894:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202abb500. 00010000:00000010:2.1:1775773851.106896:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206207e00. 00010000:00000010:2.1:1775773851.106898:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de6b000. 00010000:00000010:2.1:1775773851.106899:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da42000. 00010000:00000010:2.1:1775773851.106901:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de6b500. 00010000:00000010:2.1:1775773851.106902:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da42a00. 00010000:00000010:2.1:1775773851.106904:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb7200. 00010000:00000010:2.1:1775773851.106905:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225845200. 00010000:00000010:2.1:1775773851.106907:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de6bb00. 00010000:00000010:2.1:1775773851.106909:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da43400. 00010000:00000010:2.1:1775773851.106910:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202abbb00. 00010000:00000010:2.1:1775773851.106915:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6206207200. 00010000:00000010:2.1:1775773851.106916:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621de6bc00. 00010000:00000010:2.1:1775773851.106922:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622da43c00. 00010000:00000010:2.1:1775773851.106924:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb7d00. 00010000:00000010:2.1:1775773851.106926:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6225844400. 00000800:00000200:1.1:1775773856.060174:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773856.060178:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773856.060604:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773856.060607:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773856.061028:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773856.061031:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773861.179744:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773861.179748:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775773861.180218:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773861.180220:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775773861.180403:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773861.180405:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00010000:00000040:2.0:1775773861.340569:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f622ce91500 count: 2 00010000:00000001:2.0:1775773861.340570:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.340572:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.340573:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.340573:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.340574:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f622ce91500 count: 2 00010000:00000001:2.0:1775773861.340576:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.340577:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.340578:0:207610:0:(file.c:6054:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.340579:0:207610:0:(namei.c:211:ll_dom_lock_cancel()) Process entered 00000020:00000001:2.0:1775773861.340579:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.340581:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.340583:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.340584:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.340586:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.340587:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.340589:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.340590:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.340592:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.340593:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.340595:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.340596:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.340598:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.340599:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.340601:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.340602:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.340603:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.340604:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00020000:00000002:2.0:1775773861.340606:0:207610:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff8f622c0419b0) owner 0000000000000000/ffff8f62012d8000 00020000:00000001:2.0:1775773861.340608:0:207610:0:(lov_object.c:1110:lov_flush_composite()) Process entered 00000020:00000001:2.0:1775773861.340609:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00000002:00000001:2.0:1775773861.340610:0:207610:0:(mdc_dev.c:306:mdc_dlm_canceling()) Process entered 00000002:00000001:2.0:1775773861.340611:0:207610:0:(mdc_dev.c:255:mdc_lock_flush()) Process entered 00000008:00000001:2.0:1775773861.340612:0:207610:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775773861.340614:0:207610:0:(osc_cache.c:3200:osc_cache_wait_range()) Process entered 00000008:00000020:2.0:1775773861.340615:0:207610:0:(osc_cache.c:3237:osc_cache_wait_range()) obj ffff8f621ddb4000 ready 0|-|- wr 0|-|- rd 0|- sync file range. 00000008:00000001:2.0:1775773861.340617:0:207610:0:(osc_cache.c:3238:osc_cache_wait_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775773861.340619:0:207610:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff8f621ddb4000 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:2.0:1775773861.340621:0:207610:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000002:00000020:2.0:1775773861.340623:0:207610:0:(mdc_dev.c:262:mdc_lock_flush()) object ffff8f621ddb4000: [0 -> 18446744073709551615] 0 pages were written back. 00000002:00000001:2.0:1775773861.340624:0:207610:0:(mdc_dev.c:228:mdc_lock_discard_pages()) Process entered 00000020:00000001:2.0:1775773861.340625:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621ddb4000 -> ffff8f62258ca210 00000020:00000001:2.0:1775773861.340626:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f62258ca210 -> ffff8f62258ca210 00000020:00000001:2.0:1775773861.340627:0:207610:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775773861.340628:0:207610:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775773861.340629:0:207610:0:(vvp_io.c:1817:vvp_io_init()) [0x200002b15:0x99a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775773861.340630:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.340631:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775773861.340632:0:207610:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.340633:0:207610:0:(lov_object.c:1554:lov_io_init()) [0x200002b15:0x99a:0x0]io ffff8f6205260bc8 type 8 ignore/verify layout 1/0 00020000:00000001:2.0:1775773861.340634:0:207610:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775773861.340635:0:207610:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775773861.340636:0:207610:0:(lov_io.c:599:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775773861.340637:0:207610:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775773861.340638:0:207610:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.340639:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.340640:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775773861.340641:0:207610:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000001:2.0:1775773861.340642:0:207610:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775773861.340643:0:207610:0:(osc_cache.c:3447:osc_page_gang_lookup()) Process entered 00000008:00000001:2.0:1775773861.340644:0:207610:0:(osc_cache.c:3524:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775773861.340645:0:207610:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775773861.340646:0:207610:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775773861.340646:0:207610:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775773861.340647:0:207610:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775773861.340648:0:207610:0:(vvp_io.c:288:vvp_io_fini()) [0x200002b15:0x99a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775773861.340650:0:207610:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775773861.340650:0:207610:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000002:00000001:2.0:1775773861.340651:0:207610:0:(mdc_dev.c:245:mdc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.340652:0:207610:0:(mdc_dev.c:273:mdc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.340653:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621ddb4000 -> ffff8f62258ca210 00000020:00000001:2.0:1775773861.340654:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621ddb4000 -> ffff8f62258ca210 00000020:00000001:2.0:1775773861.340655:0:207610:0:(cl_object.c:256:cl_object_attr_update()) Process entered 00000008:00000020:2.0:1775773861.340657:0:207610:0:(osc_object.c:161:osc_attr_update()) set kms from 131072to 0 00020000:00000001:2.0:1775773861.340658:0:207610:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:2.0:1775773861.340659:0:207610:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.340660:0:207610:0:(cl_object.c:269:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.340661:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621ddb4000 -> ffff8f62258ca210 00000002:00000001:2.0:1775773861.340662:0:207610:0:(mdc_dev.c:346:mdc_dlm_canceling()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.340663:0:207610:0:(mdc_dev.c:1535:mdc_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.340665:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775773861.340666:0:207610:0:(lov_object.c:1119:lov_flush_composite()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.340667:0:207610:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff8f622c0419b0) owner 0000000000000000/ffff8f62012d8000 00000020:00000001:2.0:1775773861.340668:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.340670:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.340671:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.340672:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.340674:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.340675:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.340676:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.340678:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.340679:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.340681:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.340683:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.340684:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.340685:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.340687:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.340688:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.340690:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.340691:0:207610:0:(namei.c:229:ll_dom_lock_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.340692:0:207610:0:(file.c:6731:ll_layout_conf()) Process entered 00000020:00000001:2.0:1775773861.340692:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.340694:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.340695:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.340697:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.340698:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.340700:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.340701:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.340703:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.340705:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.340706:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.340707:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.340709:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.340710:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.340711:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.340713:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.340714:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.340715:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.340716:0:207610:0:(cl_object.c:351:cl_conf_set()) Process entered 00000080:00200000:2.0:1775773861.340717:0:207610:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b15:0x99a:0x0]: losing layout lock 00000080:00000001:2.0:1775773861.340718:0:207610:0:(pcc.c:2213:pcc_layout_invalidate()) Process entered 00000080:00000001:2.0:1775773861.340719:0:207610:0:(pcc.c:2225:pcc_layout_invalidate()) Process leaving 00020000:00000001:2.0:1775773861.340720:0:207610:0:(lov_object.c:1412:lov_conf_set()) Process entered 00020000:00000001:2.0:1775773861.340721:0:207610:0:(lov_object.c:1426:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0) 00020000:00000002:2.0:1775773861.340722:0:207610:0:(lov_object.c:1499:lov_conf_set()) [0x200002b15:0x99a:0x0] lo_layout_invalid=1 00020000:00000001:2.0:1775773861.340723:0:207610:0:(lov_object.c:1500:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.340724:0:207610:0:(cl_object.c:359:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.340725:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.340726:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.340727:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.340729:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.340730:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.340731:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.340733:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.340734:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.340735:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.340737:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.340738:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.340739:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.340741:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.340742:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.340744:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.340745:0:207610:0:(file.c:6773:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.340746:0:207610:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:2.0:1775773861.340747:0:207610:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b15:0x99a:0x0](ffff8f6226a19148) invalid 00000020:00000001:2.0:1775773861.340749:0:207610:0:(obd_sysfs.c:903:encode_fn_len()) Process filename at ffffaf0701cebca0 00000080:00002000:2.0:1775773861.340750:0:207610:0:(llite_internal.h:2024:d_lustre_invalidate()) invalidate dentry f.114 (ffff8f6209b636c0) parent ffff8f6202184798 inode ffff8f6226a19148 refc 0 00000080:00000001:2.0:1775773861.340753:0:207610:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:2.0:1775773861.340756:0:207610:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:2.0:1775773861.340758:0:207610:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:2.0:1775773861.340759:0:207610:0:(namei.c:402:ll_lock_cancel_bits()) Process leaving 00000080:00000001:2.0:1775773861.340760:0:207610:0:(namei.c:536:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775773861.340762:0:207610:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1775773861.340763:0:207610:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1775773861.340764:0:207610:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1775773861.340765:0:207610:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1775773861.340766:0:207610:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775773861.340767:0:207610:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8f622c1ec200 with handle 0xf74d1861c614870 from hash 00010000:00000001:2.0:1775773861.340769:0:207610:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1775773861.340769:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1775773861.340770:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.340771:0:207610:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1775773861.340772:0:207610:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1775773861.340773:0:207610:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:2.0:1775773861.340775:0:207610:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1775773861.340777:0:207610:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226128200/0xf74d1861c614869 lrc: 2/0,0 mode: PW/PW res: [0x200002b15:0x999:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x800028400000000 nid: local remote: 0xd88660ed39b2ceae expref: -99 pid: 214102 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00000001:2.0:1775773861.340781:0:207610:0:(namei.c:472:ll_md_blocking_ast()) Process entered 00000080:00000001:2.0:1775773861.340782:0:207610:0:(namei.c:239:ll_lock_cancel_bits()) Process entered 00000080:00000001:2.0:1775773861.340783:0:207610:0:(file.c:6026:ll_have_md_lock()) Process entered 00000080:00000040:2.0:1775773861.340784:0:207610:0:(file.c:6032:ll_have_md_lock()) trying to match res [0x200002b15:0x999:0x0] mode (null) 00000002:00000001:2.0:1775773861.340785:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.340786:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.340788:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960300 count: 2 00010000:00000001:2.0:1775773861.340789:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.340790:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.340791:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.340793:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.340794:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960300 count: 2 00010000:00000001:2.0:1775773861.340795:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.340796:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.340797:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.340798:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.340799:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960300 count: 2 00010000:00000001:2.0:1775773861.340800:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.340801:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.340802:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.340803:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.340804:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960300 count: 2 00010000:00000001:2.0:1775773861.340805:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.340806:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.340807:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.340807:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.340808:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960300 count: 2 00010000:00000001:2.0:1775773861.340809:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.340811:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.340811:0:207610:0:(file.c:6054:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.340812:0:207610:0:(namei.c:211:ll_dom_lock_cancel()) Process entered 00000020:00000001:2.0:1775773861.340813:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.340814:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.340816:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.340817:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.340819:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.340820:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.340821:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.340823:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.340824:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.340826:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.340827:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.340828:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.340830:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.340831:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.340833:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.340834:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.340835:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.340836:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00020000:00000002:2.0:1775773861.340838:0:207610:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff8f622e1964d8) owner 0000000000000000/ffff8f62012d8000 00020000:00000001:2.0:1775773861.340839:0:207610:0:(lov_object.c:1110:lov_flush_composite()) Process entered 00000020:00000001:2.0:1775773861.340840:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00000002:00000001:2.0:1775773861.340841:0:207610:0:(mdc_dev.c:306:mdc_dlm_canceling()) Process entered 00000002:00000001:2.0:1775773861.340842:0:207610:0:(mdc_dev.c:255:mdc_lock_flush()) Process entered 00000008:00000001:2.0:1775773861.340843:0:207610:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775773861.340844:0:207610:0:(osc_cache.c:3200:osc_cache_wait_range()) Process entered 00000008:00000020:2.0:1775773861.340845:0:207610:0:(osc_cache.c:3237:osc_cache_wait_range()) obj ffff8f623476e1c0 ready 0|-|- wr 0|-|- rd 0|- sync file range. 00000008:00000001:2.0:1775773861.340846:0:207610:0:(osc_cache.c:3238:osc_cache_wait_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775773861.340848:0:207610:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff8f623476e1c0 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:2.0:1775773861.340850:0:207610:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000002:00000020:2.0:1775773861.340851:0:207610:0:(mdc_dev.c:262:mdc_lock_flush()) object ffff8f623476e1c0: [0 -> 18446744073709551615] 0 pages were written back. 00000002:00000001:2.0:1775773861.340852:0:207610:0:(mdc_dev.c:228:mdc_lock_discard_pages()) Process entered 00000020:00000001:2.0:1775773861.340853:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476e1c0 -> ffff8f6202fc65f8 00000020:00000001:2.0:1775773861.340854:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f6202fc65f8 -> ffff8f6202fc65f8 00000020:00000001:2.0:1775773861.340856:0:207610:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775773861.340856:0:207610:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775773861.340857:0:207610:0:(vvp_io.c:1817:vvp_io_init()) [0x200002b15:0x999:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775773861.340859:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.340859:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775773861.340860:0:207610:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.340861:0:207610:0:(lov_object.c:1554:lov_io_init()) [0x200002b15:0x999:0x0]io ffff8f6205260bc8 type 8 ignore/verify layout 1/0 00020000:00000001:2.0:1775773861.340863:0:207610:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775773861.340864:0:207610:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775773861.340864:0:207610:0:(lov_io.c:599:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775773861.340865:0:207610:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775773861.340866:0:207610:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.340867:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.340868:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775773861.340868:0:207610:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000001:2.0:1775773861.340869:0:207610:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775773861.340870:0:207610:0:(osc_cache.c:3447:osc_page_gang_lookup()) Process entered 00000008:00000001:2.0:1775773861.340872:0:207610:0:(osc_cache.c:3524:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775773861.340873:0:207610:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775773861.340873:0:207610:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775773861.340874:0:207610:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775773861.340875:0:207610:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775773861.340876:0:207610:0:(vvp_io.c:288:vvp_io_fini()) [0x200002b15:0x999:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775773861.340878:0:207610:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775773861.340879:0:207610:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000002:00000001:2.0:1775773861.340879:0:207610:0:(mdc_dev.c:245:mdc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.340880:0:207610:0:(mdc_dev.c:273:mdc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.340881:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476e1c0 -> ffff8f6202fc65f8 00000020:00000001:2.0:1775773861.340882:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476e1c0 -> ffff8f6202fc65f8 00000020:00000001:2.0:1775773861.340883:0:207610:0:(cl_object.c:256:cl_object_attr_update()) Process entered 00000008:00000020:2.0:1775773861.340884:0:207610:0:(osc_object.c:161:osc_attr_update()) set kms from 131072to 0 00020000:00000001:2.0:1775773861.340885:0:207610:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:2.0:1775773861.340886:0:207610:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.340887:0:207610:0:(cl_object.c:269:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.340888:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476e1c0 -> ffff8f6202fc65f8 00000002:00000001:2.0:1775773861.340889:0:207610:0:(mdc_dev.c:346:mdc_dlm_canceling()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.340891:0:207610:0:(mdc_dev.c:1535:mdc_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.340892:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775773861.340893:0:207610:0:(lov_object.c:1119:lov_flush_composite()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.340894:0:207610:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff8f622e1964d8) owner 0000000000000000/ffff8f62012d8000 00000020:00000001:2.0:1775773861.340895:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.340896:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.340897:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.340899:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.340900:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.340902:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.340903:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.340905:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.340906:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.340907:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.340909:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.340910:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.340912:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.340914:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.340915:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.340917:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.340918:0:207610:0:(namei.c:229:ll_dom_lock_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.340919:0:207610:0:(file.c:6731:ll_layout_conf()) Process entered 00000020:00000001:2.0:1775773861.340920:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.340921:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.340922:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.340924:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.340925:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.340927:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.340928:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.340929:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.340931:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.340932:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.340934:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.340935:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.340936:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.340938:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.340939:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.340940:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.340941:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.340942:0:207610:0:(cl_object.c:351:cl_conf_set()) Process entered 00000080:00200000:2.0:1775773861.340943:0:207610:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b15:0x999:0x0]: losing layout lock 00000080:00000001:2.0:1775773861.340944:0:207610:0:(pcc.c:2213:pcc_layout_invalidate()) Process entered 00000080:00000001:2.0:1775773861.340945:0:207610:0:(pcc.c:2225:pcc_layout_invalidate()) Process leaving 00020000:00000001:2.0:1775773861.340946:0:207610:0:(lov_object.c:1412:lov_conf_set()) Process entered 00020000:00000001:2.0:1775773861.340947:0:207610:0:(lov_object.c:1426:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0) 00020000:00000002:2.0:1775773861.340948:0:207610:0:(lov_object.c:1499:lov_conf_set()) [0x200002b15:0x999:0x0] lo_layout_invalid=1 00020000:00000001:2.0:1775773861.340949:0:207610:0:(lov_object.c:1500:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.340950:0:207610:0:(cl_object.c:359:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.340951:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.340952:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.340953:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.340954:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.340956:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.340957:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.340959:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.340960:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.340962:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.340963:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.340964:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.340965:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.340967:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.340968:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.340969:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.340970:0:207610:0:(file.c:6773:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.340971:0:207610:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:2.0:1775773861.340972:0:207610:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b15:0x999:0x0](ffff8f621efb9148) invalid 00000020:00000001:2.0:1775773861.340974:0:207610:0:(obd_sysfs.c:903:encode_fn_len()) Process filename at ffffaf0701cebca0 00000080:00002000:2.0:1775773861.340975:0:207610:0:(llite_internal.h:2024:d_lustre_invalidate()) invalidate dentry f.113 (ffff8f62191c5a20) parent ffff8f6202184798 inode ffff8f621efb9148 refc 0 00000080:00000001:2.0:1775773861.340978:0:207610:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:2.0:1775773861.340980:0:207610:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:2.0:1775773861.340982:0:207610:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:2.0:1775773861.340983:0:207610:0:(namei.c:402:ll_lock_cancel_bits()) Process leaving 00000080:00000001:2.0:1775773861.340984:0:207610:0:(namei.c:536:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775773861.340985:0:207610:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1775773861.340986:0:207610:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1775773861.340987:0:207610:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1775773861.340988:0:207610:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1775773861.340989:0:207610:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775773861.340990:0:207610:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8f6226128200 with handle 0xf74d1861c614869 from hash 00010000:00000001:2.0:1775773861.340991:0:207610:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1775773861.340992:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1775773861.340993:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.340994:0:207610:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1775773861.340995:0:207610:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1775773861.340996:0:207610:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:2.0:1775773861.340997:0:207610:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1775773861.340999:0:207610:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226129a00/0xf74d1861c614862 lrc: 2/0,0 mode: PW/PW res: [0x200002b15:0x998:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x800028400000000 nid: local remote: 0xd88660ed39b2ce8b expref: -99 pid: 214101 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00000001:2.0:1775773861.341004:0:207610:0:(namei.c:472:ll_md_blocking_ast()) Process entered 00000080:00000001:2.0:1775773861.341005:0:207610:0:(namei.c:239:ll_lock_cancel_bits()) Process entered 00000080:00000001:2.0:1775773861.341006:0:207610:0:(file.c:6026:ll_have_md_lock()) Process entered 00000080:00000040:2.0:1775773861.341007:0:207610:0:(file.c:6032:ll_have_md_lock()) trying to match res [0x200002b15:0x998:0x0] mode (null) 00000002:00000001:2.0:1775773861.341008:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341009:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341010:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960200 count: 2 00010000:00000001:2.0:1775773861.341011:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341013:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341014:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341014:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341016:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960200 count: 2 00010000:00000001:2.0:1775773861.341017:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341018:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341019:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341020:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341020:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960200 count: 2 00010000:00000001:2.0:1775773861.341022:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341023:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341024:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341024:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341025:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960200 count: 2 00010000:00000001:2.0:1775773861.341026:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341027:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341028:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341029:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341030:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960200 count: 2 00010000:00000001:2.0:1775773861.341031:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341032:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.341033:0:207610:0:(file.c:6054:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.341034:0:207610:0:(namei.c:211:ll_dom_lock_cancel()) Process entered 00000020:00000001:2.0:1775773861.341034:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.341036:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.341037:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.341039:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.341040:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.341042:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.341044:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.341045:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.341046:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.341048:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.341049:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.341051:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.341052:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.341054:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.341055:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.341057:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.341058:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.341059:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00020000:00000002:2.0:1775773861.341060:0:207610:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff8f622e1963e0) owner 0000000000000000/ffff8f62012d8000 00020000:00000001:2.0:1775773861.341061:0:207610:0:(lov_object.c:1110:lov_flush_composite()) Process entered 00000020:00000001:2.0:1775773861.341062:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00000002:00000001:2.0:1775773861.341063:0:207610:0:(mdc_dev.c:306:mdc_dlm_canceling()) Process entered 00000002:00000001:2.0:1775773861.341064:0:207610:0:(mdc_dev.c:255:mdc_lock_flush()) Process entered 00000008:00000001:2.0:1775773861.341065:0:207610:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775773861.341066:0:207610:0:(osc_cache.c:3200:osc_cache_wait_range()) Process entered 00000008:00000020:2.0:1775773861.341067:0:207610:0:(osc_cache.c:3237:osc_cache_wait_range()) obj ffff8f623476e8c0 ready 0|-|- wr 0|-|- rd 0|- sync file range. 00000008:00000001:2.0:1775773861.341068:0:207610:0:(osc_cache.c:3238:osc_cache_wait_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775773861.341070:0:207610:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff8f623476e8c0 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:2.0:1775773861.341072:0:207610:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000002:00000020:2.0:1775773861.341073:0:207610:0:(mdc_dev.c:262:mdc_lock_flush()) object ffff8f623476e8c0: [0 -> 18446744073709551615] 0 pages were written back. 00000002:00000001:2.0:1775773861.341074:0:207610:0:(mdc_dev.c:228:mdc_lock_discard_pages()) Process entered 00000020:00000001:2.0:1775773861.341075:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476e8c0 -> ffff8f6202fc62d8 00000020:00000001:2.0:1775773861.341077:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f6202fc62d8 -> ffff8f6202fc62d8 00000020:00000001:2.0:1775773861.341078:0:207610:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775773861.341078:0:207610:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775773861.341079:0:207610:0:(vvp_io.c:1817:vvp_io_init()) [0x200002b15:0x998:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775773861.341081:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.341082:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775773861.341082:0:207610:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.341084:0:207610:0:(lov_object.c:1554:lov_io_init()) [0x200002b15:0x998:0x0]io ffff8f6205260bc8 type 8 ignore/verify layout 1/0 00020000:00000001:2.0:1775773861.341085:0:207610:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775773861.341086:0:207610:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775773861.341088:0:207610:0:(lov_io.c:599:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775773861.341089:0:207610:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775773861.341089:0:207610:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341090:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.341091:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775773861.341091:0:207610:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000001:2.0:1775773861.341092:0:207610:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775773861.341093:0:207610:0:(osc_cache.c:3447:osc_page_gang_lookup()) Process entered 00000008:00000001:2.0:1775773861.341094:0:207610:0:(osc_cache.c:3524:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775773861.341095:0:207610:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775773861.341096:0:207610:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775773861.341096:0:207610:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775773861.341097:0:207610:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775773861.341098:0:207610:0:(vvp_io.c:288:vvp_io_fini()) [0x200002b15:0x998:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775773861.341100:0:207610:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775773861.341100:0:207610:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000002:00000001:2.0:1775773861.341101:0:207610:0:(mdc_dev.c:245:mdc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341102:0:207610:0:(mdc_dev.c:273:mdc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341103:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476e8c0 -> ffff8f6202fc62d8 00000020:00000001:2.0:1775773861.341104:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476e8c0 -> ffff8f6202fc62d8 00000020:00000001:2.0:1775773861.341105:0:207610:0:(cl_object.c:256:cl_object_attr_update()) Process entered 00000008:00000020:2.0:1775773861.341106:0:207610:0:(osc_object.c:161:osc_attr_update()) set kms from 131072to 0 00020000:00000001:2.0:1775773861.341107:0:207610:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:2.0:1775773861.341108:0:207610:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341109:0:207610:0:(cl_object.c:269:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341110:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476e8c0 -> ffff8f6202fc62d8 00000002:00000001:2.0:1775773861.341111:0:207610:0:(mdc_dev.c:346:mdc_dlm_canceling()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341112:0:207610:0:(mdc_dev.c:1535:mdc_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341114:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775773861.341115:0:207610:0:(lov_object.c:1119:lov_flush_composite()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.341116:0:207610:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff8f622e1963e0) owner 0000000000000000/ffff8f62012d8000 00000020:00000001:2.0:1775773861.341117:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.341118:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.341120:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.341122:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.341123:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.341125:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.341126:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.341128:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.341129:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.341131:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.341147:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.341149:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.341151:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.341153:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.341154:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.341156:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.341157:0:207610:0:(namei.c:229:ll_dom_lock_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.341158:0:207610:0:(file.c:6731:ll_layout_conf()) Process entered 00000020:00000001:2.0:1775773861.341159:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.341161:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.341163:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.341164:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.341166:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.341168:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.341169:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.341170:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.341172:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.341173:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.341175:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.341176:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.341178:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.341179:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.341180:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.341182:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.341183:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.341184:0:207610:0:(cl_object.c:351:cl_conf_set()) Process entered 00000080:00200000:2.0:1775773861.341185:0:207610:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b15:0x998:0x0]: losing layout lock 00000080:00000001:2.0:1775773861.341187:0:207610:0:(pcc.c:2213:pcc_layout_invalidate()) Process entered 00000080:00000001:2.0:1775773861.341188:0:207610:0:(pcc.c:2225:pcc_layout_invalidate()) Process leaving 00020000:00000001:2.0:1775773861.341189:0:207610:0:(lov_object.c:1412:lov_conf_set()) Process entered 00020000:00000001:2.0:1775773861.341190:0:207610:0:(lov_object.c:1426:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0) 00020000:00000002:2.0:1775773861.341191:0:207610:0:(lov_object.c:1499:lov_conf_set()) [0x200002b15:0x998:0x0] lo_layout_invalid=1 00020000:00000001:2.0:1775773861.341193:0:207610:0:(lov_object.c:1500:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341194:0:207610:0:(cl_object.c:359:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.341195:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.341196:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.341197:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.341199:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.341200:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.341201:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.341203:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.341204:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.341205:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.341207:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.341208:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.341209:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.341210:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.341212:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.341214:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.341215:0:207610:0:(file.c:6773:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.341216:0:207610:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:2.0:1775773861.341217:0:207610:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b15:0x998:0x0](ffff8f621efbec08) invalid 00000020:00000001:2.0:1775773861.341219:0:207610:0:(obd_sysfs.c:903:encode_fn_len()) Process filename at ffffaf0701cebca0 00000080:00002000:2.0:1775773861.341220:0:207610:0:(llite_internal.h:2024:d_lustre_invalidate()) invalidate dentry f.112 (ffff8f62186ac1b0) parent ffff8f6202184798 inode ffff8f621efbec08 refc 0 00000080:00000001:2.0:1775773861.341223:0:207610:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:2.0:1775773861.341225:0:207610:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:2.0:1775773861.341227:0:207610:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:2.0:1775773861.341228:0:207610:0:(namei.c:402:ll_lock_cancel_bits()) Process leaving 00000080:00000001:2.0:1775773861.341229:0:207610:0:(namei.c:536:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775773861.341231:0:207610:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1775773861.341232:0:207610:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1775773861.341233:0:207610:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1775773861.341234:0:207610:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1775773861.341236:0:207610:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775773861.341237:0:207610:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8f6226129a00 with handle 0xf74d1861c614862 from hash 00010000:00000001:2.0:1775773861.341239:0:207610:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1775773861.341240:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1775773861.341240:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.341241:0:207610:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1775773861.341242:0:207610:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1775773861.341243:0:207610:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:2.0:1775773861.341244:0:207610:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1775773861.341246:0:207610:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226129000/0xf74d1861c61485b lrc: 2/0,0 mode: PW/PW res: [0x200002b15:0x997:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x800028400000000 nid: local remote: 0xd88660ed39b2ce68 expref: -99 pid: 214100 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00000001:2.0:1775773861.341251:0:207610:0:(namei.c:472:ll_md_blocking_ast()) Process entered 00000080:00000001:2.0:1775773861.341252:0:207610:0:(namei.c:239:ll_lock_cancel_bits()) Process entered 00000080:00000001:2.0:1775773861.341253:0:207610:0:(file.c:6026:ll_have_md_lock()) Process entered 00000080:00000040:2.0:1775773861.341253:0:207610:0:(file.c:6032:ll_have_md_lock()) trying to match res [0x200002b15:0x997:0x0] mode (null) 00000002:00000001:2.0:1775773861.341255:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341256:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341257:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960d00 count: 2 00010000:00000001:2.0:1775773861.341259:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341261:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341262:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341263:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341264:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960d00 count: 2 00010000:00000001:2.0:1775773861.341265:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341266:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341267:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341268:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341269:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960d00 count: 2 00010000:00000001:2.0:1775773861.341270:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341271:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341272:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341272:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341273:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960d00 count: 2 00010000:00000001:2.0:1775773861.341275:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341276:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341277:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341278:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341279:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960d00 count: 2 00010000:00000001:2.0:1775773861.341280:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341281:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.341282:0:207610:0:(file.c:6054:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.341283:0:207610:0:(namei.c:211:ll_dom_lock_cancel()) Process entered 00000020:00000001:2.0:1775773861.341284:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.341285:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.341287:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.341288:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.341289:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.341291:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.341292:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.341294:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.341295:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.341296:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.341298:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.341299:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.341301:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.341302:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.341304:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.341305:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.341306:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.341307:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00020000:00000002:2.0:1775773861.341308:0:207610:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff8f622e1966c8) owner 0000000000000000/ffff8f62012d8000 00020000:00000001:2.0:1775773861.341310:0:207610:0:(lov_object.c:1110:lov_flush_composite()) Process entered 00000020:00000001:2.0:1775773861.341310:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00000002:00000001:2.0:1775773861.341312:0:207610:0:(mdc_dev.c:306:mdc_dlm_canceling()) Process entered 00000002:00000001:2.0:1775773861.341313:0:207610:0:(mdc_dev.c:255:mdc_lock_flush()) Process entered 00000008:00000001:2.0:1775773861.341313:0:207610:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775773861.341315:0:207610:0:(osc_cache.c:3200:osc_cache_wait_range()) Process entered 00000008:00000020:2.0:1775773861.341316:0:207610:0:(osc_cache.c:3237:osc_cache_wait_range()) obj ffff8f623476f340 ready 0|-|- wr 0|-|- rd 0|- sync file range. 00000008:00000001:2.0:1775773861.341318:0:207610:0:(osc_cache.c:3238:osc_cache_wait_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775773861.341320:0:207610:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff8f623476f340 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:2.0:1775773861.341323:0:207610:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000002:00000020:2.0:1775773861.341324:0:207610:0:(mdc_dev.c:262:mdc_lock_flush()) object ffff8f623476f340: [0 -> 18446744073709551615] 0 pages were written back. 00000002:00000001:2.0:1775773861.341325:0:207610:0:(mdc_dev.c:228:mdc_lock_discard_pages()) Process entered 00000020:00000001:2.0:1775773861.341326:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476f340 -> ffff8f6202fc6aa8 00000020:00000001:2.0:1775773861.341327:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f6202fc6aa8 -> ffff8f6202fc6aa8 00000020:00000001:2.0:1775773861.341328:0:207610:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775773861.341329:0:207610:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775773861.341330:0:207610:0:(vvp_io.c:1817:vvp_io_init()) [0x200002b15:0x997:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775773861.341331:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.341332:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775773861.341332:0:207610:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.341334:0:207610:0:(lov_object.c:1554:lov_io_init()) [0x200002b15:0x997:0x0]io ffff8f6205260bc8 type 8 ignore/verify layout 1/0 00020000:00000001:2.0:1775773861.341335:0:207610:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775773861.341336:0:207610:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775773861.341337:0:207610:0:(lov_io.c:599:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775773861.341338:0:207610:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775773861.341339:0:207610:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341340:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.341340:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775773861.341341:0:207610:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000001:2.0:1775773861.341342:0:207610:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775773861.341343:0:207610:0:(osc_cache.c:3447:osc_page_gang_lookup()) Process entered 00000008:00000001:2.0:1775773861.341344:0:207610:0:(osc_cache.c:3524:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775773861.341345:0:207610:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775773861.341346:0:207610:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775773861.341347:0:207610:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775773861.341347:0:207610:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775773861.341349:0:207610:0:(vvp_io.c:288:vvp_io_fini()) [0x200002b15:0x997:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775773861.341350:0:207610:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775773861.341351:0:207610:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000002:00000001:2.0:1775773861.341351:0:207610:0:(mdc_dev.c:245:mdc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341352:0:207610:0:(mdc_dev.c:273:mdc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341353:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476f340 -> ffff8f6202fc6aa8 00000020:00000001:2.0:1775773861.341354:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476f340 -> ffff8f6202fc6aa8 00000020:00000001:2.0:1775773861.341355:0:207610:0:(cl_object.c:256:cl_object_attr_update()) Process entered 00000008:00000020:2.0:1775773861.341357:0:207610:0:(osc_object.c:161:osc_attr_update()) set kms from 131072to 0 00020000:00000001:2.0:1775773861.341358:0:207610:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:2.0:1775773861.341359:0:207610:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341360:0:207610:0:(cl_object.c:269:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341361:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476f340 -> ffff8f6202fc6aa8 00000002:00000001:2.0:1775773861.341363:0:207610:0:(mdc_dev.c:346:mdc_dlm_canceling()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341364:0:207610:0:(mdc_dev.c:1535:mdc_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341365:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775773861.341366:0:207610:0:(lov_object.c:1119:lov_flush_composite()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.341367:0:207610:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff8f622e1966c8) owner 0000000000000000/ffff8f62012d8000 00000020:00000001:2.0:1775773861.341369:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.341370:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.341371:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.341372:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.341374:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.341375:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.341377:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.341378:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.341379:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.341381:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.341382:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.341384:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.341385:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.341387:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.341389:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.341390:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.341391:0:207610:0:(namei.c:229:ll_dom_lock_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.341392:0:207610:0:(file.c:6731:ll_layout_conf()) Process entered 00000020:00000001:2.0:1775773861.341393:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.341394:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.341395:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.341396:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.341398:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.341399:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.341401:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.341402:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.341404:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.341405:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.341407:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.341408:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.341410:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.341411:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.341412:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.341413:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.341414:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.341415:0:207610:0:(cl_object.c:351:cl_conf_set()) Process entered 00000080:00200000:2.0:1775773861.341416:0:207610:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b15:0x997:0x0]: losing layout lock 00000080:00000001:2.0:1775773861.341417:0:207610:0:(pcc.c:2213:pcc_layout_invalidate()) Process entered 00000080:00000001:2.0:1775773861.341418:0:207610:0:(pcc.c:2225:pcc_layout_invalidate()) Process leaving 00020000:00000001:2.0:1775773861.341419:0:207610:0:(lov_object.c:1412:lov_conf_set()) Process entered 00020000:00000001:2.0:1775773861.341420:0:207610:0:(lov_object.c:1426:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0) 00020000:00000002:2.0:1775773861.341421:0:207610:0:(lov_object.c:1499:lov_conf_set()) [0x200002b15:0x997:0x0] lo_layout_invalid=1 00020000:00000001:2.0:1775773861.341422:0:207610:0:(lov_object.c:1500:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341423:0:207610:0:(cl_object.c:359:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.341424:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.341425:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.341426:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.341427:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.341429:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.341430:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.341431:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.341432:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.341434:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.341435:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.341437:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.341438:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.341440:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.341441:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.341442:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.341444:0:207610:0:(file.c:6773:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.341445:0:207610:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:2.0:1775773861.341446:0:207610:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b15:0x997:0x0](ffff8f621efb80c8) invalid 00000020:00000001:2.0:1775773861.341448:0:207610:0:(obd_sysfs.c:903:encode_fn_len()) Process filename at ffffaf0701cebca0 00000080:00002000:2.0:1775773861.341449:0:207610:0:(llite_internal.h:2024:d_lustre_invalidate()) invalidate dentry f.111 (ffff8f62186acbd0) parent ffff8f6202184798 inode ffff8f621efb80c8 refc 0 00000080:00000001:2.0:1775773861.341451:0:207610:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:2.0:1775773861.341453:0:207610:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:2.0:1775773861.341455:0:207610:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:2.0:1775773861.341456:0:207610:0:(namei.c:402:ll_lock_cancel_bits()) Process leaving 00000080:00000001:2.0:1775773861.341457:0:207610:0:(namei.c:536:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775773861.341458:0:207610:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1775773861.341459:0:207610:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1775773861.341460:0:207610:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1775773861.341461:0:207610:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1775773861.341463:0:207610:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775773861.341464:0:207610:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8f6226129000 with handle 0xf74d1861c61485b from hash 00010000:00000001:2.0:1775773861.341465:0:207610:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1775773861.341466:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1775773861.341467:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.341468:0:207610:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1775773861.341481:0:207610:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1775773861.341482:0:207610:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:2.0:1775773861.341484:0:207610:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1775773861.341486:0:207610:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226129e00/0xf74d1861c614854 lrc: 2/0,0 mode: PW/PW res: [0x200002b15:0x996:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x800028400000000 nid: local remote: 0xd88660ed39b2ce45 expref: -99 pid: 214099 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00000001:2.0:1775773861.341491:0:207610:0:(namei.c:472:ll_md_blocking_ast()) Process entered 00000080:00000001:2.0:1775773861.341492:0:207610:0:(namei.c:239:ll_lock_cancel_bits()) Process entered 00000080:00000001:2.0:1775773861.341493:0:207610:0:(file.c:6026:ll_have_md_lock()) Process entered 00000080:00000040:2.0:1775773861.341494:0:207610:0:(file.c:6032:ll_have_md_lock()) trying to match res [0x200002b15:0x996:0x0] mode (null) 00000002:00000001:2.0:1775773861.341495:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341496:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341497:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f622ce91b00 count: 2 00010000:00000001:2.0:1775773861.341498:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341500:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341501:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341502:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341503:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f622ce91b00 count: 2 00010000:00000001:2.0:1775773861.341504:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341506:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341507:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341508:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341509:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f622ce91b00 count: 2 00010000:00000001:2.0:1775773861.341510:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341511:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341512:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341512:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341513:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f622ce91b00 count: 2 00010000:00000001:2.0:1775773861.341514:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341515:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341516:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341517:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341518:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f622ce91b00 count: 2 00010000:00000001:2.0:1775773861.341519:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341520:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.341521:0:207610:0:(file.c:6054:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.341522:0:207610:0:(namei.c:211:ll_dom_lock_cancel()) Process entered 00000020:00000001:2.0:1775773861.341523:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.341524:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.341526:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.341527:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.341529:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.341530:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.341532:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.341533:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.341535:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.341536:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.341537:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.341539:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.341540:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.341541:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.341543:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.341545:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.341546:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.341547:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00020000:00000002:2.0:1775773861.341549:0:207610:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff8f622c041ba0) owner 0000000000000000/ffff8f62012d8000 00020000:00000001:2.0:1775773861.341551:0:207610:0:(lov_object.c:1110:lov_flush_composite()) Process entered 00000020:00000001:2.0:1775773861.341552:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00000002:00000001:2.0:1775773861.341553:0:207610:0:(mdc_dev.c:306:mdc_dlm_canceling()) Process entered 00000002:00000001:2.0:1775773861.341554:0:207610:0:(mdc_dev.c:255:mdc_lock_flush()) Process entered 00000008:00000001:2.0:1775773861.341555:0:207610:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775773861.341556:0:207610:0:(osc_cache.c:3200:osc_cache_wait_range()) Process entered 00000008:00000020:2.0:1775773861.341558:0:207610:0:(osc_cache.c:3237:osc_cache_wait_range()) obj ffff8f621ddb56c0 ready 0|-|- wr 0|-|- rd 0|- sync file range. 00000008:00000001:2.0:1775773861.341559:0:207610:0:(osc_cache.c:3238:osc_cache_wait_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775773861.341561:0:207610:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff8f621ddb56c0 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:2.0:1775773861.341563:0:207610:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000002:00000020:2.0:1775773861.341564:0:207610:0:(mdc_dev.c:262:mdc_lock_flush()) object ffff8f621ddb56c0: [0 -> 18446744073709551615] 0 pages were written back. 00000002:00000001:2.0:1775773861.341566:0:207610:0:(mdc_dev.c:228:mdc_lock_discard_pages()) Process entered 00000020:00000001:2.0:1775773861.341566:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621ddb56c0 -> ffff8f62258ca850 00000020:00000001:2.0:1775773861.341567:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f62258ca850 -> ffff8f62258ca850 00000020:00000001:2.0:1775773861.341568:0:207610:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775773861.341569:0:207610:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775773861.341570:0:207610:0:(vvp_io.c:1817:vvp_io_init()) [0x200002b15:0x996:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775773861.341572:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.341572:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775773861.341573:0:207610:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.341574:0:207610:0:(lov_object.c:1554:lov_io_init()) [0x200002b15:0x996:0x0]io ffff8f6205260bc8 type 8 ignore/verify layout 1/0 00020000:00000001:2.0:1775773861.341576:0:207610:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775773861.341576:0:207610:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775773861.341578:0:207610:0:(lov_io.c:599:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775773861.341579:0:207610:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775773861.341579:0:207610:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341580:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.341581:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775773861.341581:0:207610:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000001:2.0:1775773861.341582:0:207610:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775773861.341583:0:207610:0:(osc_cache.c:3447:osc_page_gang_lookup()) Process entered 00000008:00000001:2.0:1775773861.341585:0:207610:0:(osc_cache.c:3524:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775773861.341586:0:207610:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775773861.341587:0:207610:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775773861.341587:0:207610:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775773861.341588:0:207610:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775773861.341589:0:207610:0:(vvp_io.c:288:vvp_io_fini()) [0x200002b15:0x996:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775773861.341591:0:207610:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775773861.341592:0:207610:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000002:00000001:2.0:1775773861.341592:0:207610:0:(mdc_dev.c:245:mdc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341593:0:207610:0:(mdc_dev.c:273:mdc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341594:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621ddb56c0 -> ffff8f62258ca850 00000020:00000001:2.0:1775773861.341595:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621ddb56c0 -> ffff8f62258ca850 00000020:00000001:2.0:1775773861.341596:0:207610:0:(cl_object.c:256:cl_object_attr_update()) Process entered 00000008:00000020:2.0:1775773861.341597:0:207610:0:(osc_object.c:161:osc_attr_update()) set kms from 131072to 0 00020000:00000001:2.0:1775773861.341598:0:207610:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:2.0:1775773861.341599:0:207610:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341600:0:207610:0:(cl_object.c:269:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341601:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621ddb56c0 -> ffff8f62258ca850 00000002:00000001:2.0:1775773861.341602:0:207610:0:(mdc_dev.c:346:mdc_dlm_canceling()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341603:0:207610:0:(mdc_dev.c:1535:mdc_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341604:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775773861.341605:0:207610:0:(lov_object.c:1119:lov_flush_composite()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.341607:0:207610:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff8f622c041ba0) owner 0000000000000000/ffff8f62012d8000 00000020:00000001:2.0:1775773861.341608:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.341609:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.341610:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.341612:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.341614:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.341615:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.341617:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.341618:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.341620:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.341621:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.341623:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.341624:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.341626:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.341628:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.341629:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.341631:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.341632:0:207610:0:(namei.c:229:ll_dom_lock_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.341633:0:207610:0:(file.c:6731:ll_layout_conf()) Process entered 00000020:00000001:2.0:1775773861.341634:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.341635:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.341637:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.341638:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.341640:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.341641:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.341643:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.341644:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.341645:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.341647:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.341648:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.341649:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.341651:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.341652:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.341653:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.341654:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.341655:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.341656:0:207610:0:(cl_object.c:351:cl_conf_set()) Process entered 00000080:00200000:2.0:1775773861.341657:0:207610:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b15:0x996:0x0]: losing layout lock 00000080:00000001:2.0:1775773861.341658:0:207610:0:(pcc.c:2213:pcc_layout_invalidate()) Process entered 00000080:00000001:2.0:1775773861.341659:0:207610:0:(pcc.c:2225:pcc_layout_invalidate()) Process leaving 00020000:00000001:2.0:1775773861.341660:0:207610:0:(lov_object.c:1412:lov_conf_set()) Process entered 00020000:00000001:2.0:1775773861.341661:0:207610:0:(lov_object.c:1426:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0) 00020000:00000002:2.0:1775773861.341662:0:207610:0:(lov_object.c:1499:lov_conf_set()) [0x200002b15:0x996:0x0] lo_layout_invalid=1 00020000:00000001:2.0:1775773861.341664:0:207610:0:(lov_object.c:1500:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341665:0:207610:0:(cl_object.c:359:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.341666:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.341667:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.341668:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.341670:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.341671:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.341673:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.341674:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.341676:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.341677:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.341678:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.341680:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.341681:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.341682:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.341684:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.341685:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.341686:0:207610:0:(file.c:6773:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.341687:0:207610:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:2.0:1775773861.341688:0:207610:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b15:0x996:0x0](ffff8f6226a1ec08) invalid 00000020:00000001:2.0:1775773861.341689:0:207610:0:(obd_sysfs.c:903:encode_fn_len()) Process filename at ffffaf0701cebca0 00000080:00002000:2.0:1775773861.341690:0:207610:0:(llite_internal.h:2024:d_lustre_invalidate()) invalidate dentry f.110 (ffff8f62186acca8) parent ffff8f6202184798 inode ffff8f6226a1ec08 refc 0 00000080:00000001:2.0:1775773861.341693:0:207610:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:2.0:1775773861.341695:0:207610:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:2.0:1775773861.341697:0:207610:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:2.0:1775773861.341698:0:207610:0:(namei.c:402:ll_lock_cancel_bits()) Process leaving 00000080:00000001:2.0:1775773861.341699:0:207610:0:(namei.c:536:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775773861.341700:0:207610:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1775773861.341701:0:207610:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1775773861.341702:0:207610:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1775773861.341703:0:207610:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1775773861.341704:0:207610:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775773861.341705:0:207610:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8f6226129e00 with handle 0xf74d1861c614854 from hash 00010000:00000001:2.0:1775773861.341706:0:207610:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1775773861.341707:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1775773861.341708:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.341709:0:207610:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1775773861.341710:0:207610:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1775773861.341711:0:207610:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:2.0:1775773861.341713:0:207610:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1775773861.341715:0:207610:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f621839e800/0xf74d1861c61484d lrc: 2/0,0 mode: PW/PW res: [0x200002b15:0x995:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x800028400000000 nid: local remote: 0xd88660ed39b2ce22 expref: -99 pid: 214098 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00000001:2.0:1775773861.341719:0:207610:0:(namei.c:472:ll_md_blocking_ast()) Process entered 00000080:00000001:2.0:1775773861.341721:0:207610:0:(namei.c:239:ll_lock_cancel_bits()) Process entered 00000080:00000001:2.0:1775773861.341722:0:207610:0:(file.c:6026:ll_have_md_lock()) Process entered 00000080:00000040:2.0:1775773861.341723:0:207610:0:(file.c:6032:ll_have_md_lock()) trying to match res [0x200002b15:0x995:0x0] mode (null) 00000002:00000001:2.0:1775773861.341724:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341724:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341726:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960400 count: 2 00010000:00000001:2.0:1775773861.341727:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341728:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341729:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341730:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341731:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960400 count: 2 00010000:00000001:2.0:1775773861.341733:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341734:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341735:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341736:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341737:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960400 count: 2 00010000:00000001:2.0:1775773861.341738:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341739:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341740:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341740:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341741:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960400 count: 2 00010000:00000001:2.0:1775773861.341742:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341743:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341744:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341745:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341746:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960400 count: 2 00010000:00000001:2.0:1775773861.341747:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341748:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.341749:0:207610:0:(file.c:6054:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.341750:0:207610:0:(namei.c:211:ll_dom_lock_cancel()) Process entered 00000020:00000001:2.0:1775773861.341750:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.341752:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.341753:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.341755:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.341757:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.341758:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.341760:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.341761:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.341763:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.341764:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.341766:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.341767:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.341769:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.341770:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.341772:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.341773:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.341774:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.341775:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00020000:00000002:2.0:1775773861.341776:0:207610:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff8f622e1967c0) owner 0000000000000000/ffff8f62012d8000 00020000:00000001:2.0:1775773861.341778:0:207610:0:(lov_object.c:1110:lov_flush_composite()) Process entered 00000020:00000001:2.0:1775773861.341778:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00000002:00000001:2.0:1775773861.341780:0:207610:0:(mdc_dev.c:306:mdc_dlm_canceling()) Process entered 00000002:00000001:2.0:1775773861.341781:0:207610:0:(mdc_dev.c:255:mdc_lock_flush()) Process entered 00000008:00000001:2.0:1775773861.341781:0:207610:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775773861.341782:0:207610:0:(osc_cache.c:3200:osc_cache_wait_range()) Process entered 00000008:00000020:2.0:1775773861.341783:0:207610:0:(osc_cache.c:3237:osc_cache_wait_range()) obj ffff8f623476e000 ready 0|-|- wr 0|-|- rd 0|- sync file range. 00000008:00000001:2.0:1775773861.341785:0:207610:0:(osc_cache.c:3238:osc_cache_wait_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775773861.341786:0:207610:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff8f623476e000 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:2.0:1775773861.341788:0:207610:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000002:00000020:2.0:1775773861.341789:0:207610:0:(mdc_dev.c:262:mdc_lock_flush()) object ffff8f623476e000: [0 -> 18446744073709551615] 0 pages were written back. 00000002:00000001:2.0:1775773861.341791:0:207610:0:(mdc_dev.c:228:mdc_lock_discard_pages()) Process entered 00000020:00000001:2.0:1775773861.341792:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476e000 -> ffff8f6202fc6918 00000020:00000001:2.0:1775773861.341793:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f6202fc6918 -> ffff8f6202fc6918 00000020:00000001:2.0:1775773861.341794:0:207610:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775773861.341794:0:207610:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775773861.341795:0:207610:0:(vvp_io.c:1817:vvp_io_init()) [0x200002b15:0x995:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775773861.341797:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.341798:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775773861.341799:0:207610:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.341800:0:207610:0:(lov_object.c:1554:lov_io_init()) [0x200002b15:0x995:0x0]io ffff8f6205260bc8 type 8 ignore/verify layout 1/0 00020000:00000001:2.0:1775773861.341802:0:207610:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775773861.341803:0:207610:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775773861.341804:0:207610:0:(lov_io.c:599:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775773861.341805:0:207610:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775773861.341806:0:207610:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341806:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.341807:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775773861.341808:0:207610:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000001:2.0:1775773861.341808:0:207610:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775773861.341809:0:207610:0:(osc_cache.c:3447:osc_page_gang_lookup()) Process entered 00000008:00000001:2.0:1775773861.341810:0:207610:0:(osc_cache.c:3524:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775773861.341811:0:207610:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775773861.341812:0:207610:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775773861.341813:0:207610:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775773861.341813:0:207610:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775773861.341814:0:207610:0:(vvp_io.c:288:vvp_io_fini()) [0x200002b15:0x995:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775773861.341816:0:207610:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775773861.341817:0:207610:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000002:00000001:2.0:1775773861.341817:0:207610:0:(mdc_dev.c:245:mdc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341818:0:207610:0:(mdc_dev.c:273:mdc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341819:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476e000 -> ffff8f6202fc6918 00000020:00000001:2.0:1775773861.341820:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476e000 -> ffff8f6202fc6918 00000020:00000001:2.0:1775773861.341821:0:207610:0:(cl_object.c:256:cl_object_attr_update()) Process entered 00000008:00000020:2.0:1775773861.341822:0:207610:0:(osc_object.c:161:osc_attr_update()) set kms from 131072to 0 00020000:00000001:2.0:1775773861.341823:0:207610:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:2.0:1775773861.341824:0:207610:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341825:0:207610:0:(cl_object.c:269:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341826:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476e000 -> ffff8f6202fc6918 00000002:00000001:2.0:1775773861.341827:0:207610:0:(mdc_dev.c:346:mdc_dlm_canceling()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341828:0:207610:0:(mdc_dev.c:1535:mdc_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341830:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775773861.341831:0:207610:0:(lov_object.c:1119:lov_flush_composite()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.341832:0:207610:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff8f622e1967c0) owner 0000000000000000/ffff8f62012d8000 00000020:00000001:2.0:1775773861.341833:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.341835:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.341836:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.341838:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.341840:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.341841:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.341842:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.341844:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.341845:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.341847:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.341848:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.341850:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.341851:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.341852:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.341854:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.341855:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.341856:0:207610:0:(namei.c:229:ll_dom_lock_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.341857:0:207610:0:(file.c:6731:ll_layout_conf()) Process entered 00000020:00000001:2.0:1775773861.341858:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.341859:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.341860:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.341861:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.341863:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.341864:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.341865:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.341867:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.341868:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.341869:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.341871:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.341872:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.341874:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.341875:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.341877:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.341878:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.341879:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.341881:0:207610:0:(cl_object.c:351:cl_conf_set()) Process entered 00000080:00200000:2.0:1775773861.341882:0:207610:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b15:0x995:0x0]: losing layout lock 00000080:00000001:2.0:1775773861.341883:0:207610:0:(pcc.c:2213:pcc_layout_invalidate()) Process entered 00000080:00000001:2.0:1775773861.341884:0:207610:0:(pcc.c:2225:pcc_layout_invalidate()) Process leaving 00020000:00000001:2.0:1775773861.341885:0:207610:0:(lov_object.c:1412:lov_conf_set()) Process entered 00020000:00000001:2.0:1775773861.341886:0:207610:0:(lov_object.c:1426:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0) 00020000:00000002:2.0:1775773861.341887:0:207610:0:(lov_object.c:1499:lov_conf_set()) [0x200002b15:0x995:0x0] lo_layout_invalid=1 00020000:00000001:2.0:1775773861.341888:0:207610:0:(lov_object.c:1500:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.341889:0:207610:0:(cl_object.c:359:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.341890:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.341891:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.341892:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.341894:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.341895:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.341896:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.341898:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.341899:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.341900:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.341901:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.341903:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.341904:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.341906:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.341907:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.341908:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.341910:0:207610:0:(file.c:6773:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.341911:0:207610:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:2.0:1775773861.341912:0:207610:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b15:0x995:0x0](ffff8f621efbc2c8) invalid 00000020:00000001:2.0:1775773861.341913:0:207610:0:(obd_sysfs.c:903:encode_fn_len()) Process filename at ffffaf0701cebca0 00000080:00002000:2.0:1775773861.341914:0:207610:0:(llite_internal.h:2024:d_lustre_invalidate()) invalidate dentry f.109 (ffff8f62186ac798) parent ffff8f6202184798 inode ffff8f621efbc2c8 refc 0 00000080:00000001:2.0:1775773861.341917:0:207610:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:2.0:1775773861.341919:0:207610:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:2.0:1775773861.341921:0:207610:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:2.0:1775773861.341922:0:207610:0:(namei.c:402:ll_lock_cancel_bits()) Process leaving 00000080:00000001:2.0:1775773861.341922:0:207610:0:(namei.c:536:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775773861.341924:0:207610:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1775773861.341925:0:207610:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1775773861.341926:0:207610:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1775773861.341927:0:207610:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1775773861.341928:0:207610:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775773861.341930:0:207610:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8f621839e800 with handle 0xf74d1861c61484d from hash 00010000:00000001:2.0:1775773861.341931:0:207610:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1775773861.341932:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1775773861.341933:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.341933:0:207610:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1775773861.341934:0:207610:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1775773861.341935:0:207610:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:2.0:1775773861.341937:0:207610:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1775773861.341939:0:207610:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62265d6200/0xf74d1861c614846 lrc: 2/0,0 mode: PW/PW res: [0x200002b15:0x994:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x800028400000000 nid: local remote: 0xd88660ed39b2cdff expref: -99 pid: 214097 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00000001:2.0:1775773861.341943:0:207610:0:(namei.c:472:ll_md_blocking_ast()) Process entered 00000080:00000001:2.0:1775773861.341944:0:207610:0:(namei.c:239:ll_lock_cancel_bits()) Process entered 00000080:00000001:2.0:1775773861.341945:0:207610:0:(file.c:6026:ll_have_md_lock()) Process entered 00000080:00000040:2.0:1775773861.341946:0:207610:0:(file.c:6032:ll_have_md_lock()) trying to match res [0x200002b15:0x994:0x0] mode (null) 00000002:00000001:2.0:1775773861.341947:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341948:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341949:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6206bb7300 count: 2 00010000:00000001:2.0:1775773861.341951:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341952:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341953:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341954:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341955:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6206bb7300 count: 2 00010000:00000001:2.0:1775773861.341956:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341958:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341959:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341960:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341961:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6206bb7300 count: 2 00010000:00000001:2.0:1775773861.341962:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341963:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341964:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341965:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341965:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6206bb7300 count: 2 00010000:00000001:2.0:1775773861.341967:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341968:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.341969:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.341970:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.341971:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6206bb7300 count: 2 00010000:00000001:2.0:1775773861.341972:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.341973:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.341974:0:207610:0:(file.c:6054:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.341975:0:207610:0:(namei.c:211:ll_dom_lock_cancel()) Process entered 00000020:00000001:2.0:1775773861.341976:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.341977:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.341979:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.341980:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.341981:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.341983:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.341984:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.341985:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.341987:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.341988:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.341989:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.341991:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.341992:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.341993:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.341995:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.341996:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.341997:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.341999:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00020000:00000002:2.0:1775773861.342000:0:207610:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff8f6205794ba0) owner 0000000000000000/ffff8f62012d8000 00020000:00000001:2.0:1775773861.342002:0:207610:0:(lov_object.c:1110:lov_flush_composite()) Process entered 00000020:00000001:2.0:1775773861.342003:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00000002:00000001:2.0:1775773861.342004:0:207610:0:(mdc_dev.c:306:mdc_dlm_canceling()) Process entered 00000002:00000001:2.0:1775773861.342005:0:207610:0:(mdc_dev.c:255:mdc_lock_flush()) Process entered 00000008:00000001:2.0:1775773861.342006:0:207610:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775773861.342006:0:207610:0:(osc_cache.c:3200:osc_cache_wait_range()) Process entered 00000008:00000020:2.0:1775773861.342008:0:207610:0:(osc_cache.c:3237:osc_cache_wait_range()) obj ffff8f622d9aca80 ready 0|-|- wr 0|-|- rd 0|- sync file range. 00000008:00000001:2.0:1775773861.342010:0:207610:0:(osc_cache.c:3238:osc_cache_wait_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775773861.342011:0:207610:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff8f622d9aca80 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:2.0:1775773861.342013:0:207610:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000002:00000020:2.0:1775773861.342015:0:207610:0:(mdc_dev.c:262:mdc_lock_flush()) object ffff8f622d9aca80: [0 -> 18446744073709551615] 0 pages were written back. 00000002:00000001:2.0:1775773861.342016:0:207610:0:(mdc_dev.c:228:mdc_lock_discard_pages()) Process entered 00000020:00000001:2.0:1775773861.342017:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f622d9aca80 -> ffff8f622e29be90 00000020:00000001:2.0:1775773861.342018:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f622e29be90 -> ffff8f622e29be90 00000020:00000001:2.0:1775773861.342019:0:207610:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775773861.342019:0:207610:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775773861.342020:0:207610:0:(vvp_io.c:1817:vvp_io_init()) [0x200002b15:0x994:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775773861.342022:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.342022:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775773861.342023:0:207610:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.342024:0:207610:0:(lov_object.c:1554:lov_io_init()) [0x200002b15:0x994:0x0]io ffff8f6205260bc8 type 8 ignore/verify layout 1/0 00020000:00000001:2.0:1775773861.342026:0:207610:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775773861.342026:0:207610:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775773861.342027:0:207610:0:(lov_io.c:599:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775773861.342028:0:207610:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775773861.342029:0:207610:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342030:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.342031:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775773861.342031:0:207610:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000001:2.0:1775773861.342032:0:207610:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775773861.342033:0:207610:0:(osc_cache.c:3447:osc_page_gang_lookup()) Process entered 00000008:00000001:2.0:1775773861.342034:0:207610:0:(osc_cache.c:3524:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775773861.342035:0:207610:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775773861.342036:0:207610:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775773861.342037:0:207610:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775773861.342038:0:207610:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775773861.342039:0:207610:0:(vvp_io.c:288:vvp_io_fini()) [0x200002b15:0x994:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775773861.342041:0:207610:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775773861.342041:0:207610:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000002:00000001:2.0:1775773861.342042:0:207610:0:(mdc_dev.c:245:mdc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342043:0:207610:0:(mdc_dev.c:273:mdc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342044:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f622d9aca80 -> ffff8f622e29be90 00000020:00000001:2.0:1775773861.342045:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f622d9aca80 -> ffff8f622e29be90 00000020:00000001:2.0:1775773861.342046:0:207610:0:(cl_object.c:256:cl_object_attr_update()) Process entered 00000008:00000020:2.0:1775773861.342047:0:207610:0:(osc_object.c:161:osc_attr_update()) set kms from 131072to 0 00020000:00000001:2.0:1775773861.342049:0:207610:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:2.0:1775773861.342050:0:207610:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342051:0:207610:0:(cl_object.c:269:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342052:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f622d9aca80 -> ffff8f622e29be90 00000002:00000001:2.0:1775773861.342053:0:207610:0:(mdc_dev.c:346:mdc_dlm_canceling()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342054:0:207610:0:(mdc_dev.c:1535:mdc_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342055:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775773861.342056:0:207610:0:(lov_object.c:1119:lov_flush_composite()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.342058:0:207610:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff8f6205794ba0) owner 0000000000000000/ffff8f62012d8000 00000020:00000001:2.0:1775773861.342059:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.342060:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.342062:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.342064:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.342065:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.342066:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.342067:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.342069:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.342070:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.342072:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.342073:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.342075:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.342076:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.342078:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.342079:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.342081:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.342082:0:207610:0:(namei.c:229:ll_dom_lock_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.342083:0:207610:0:(file.c:6731:ll_layout_conf()) Process entered 00000020:00000001:2.0:1775773861.342083:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.342085:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.342086:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.342087:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.342089:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.342090:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.342091:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.342093:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.342094:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.342096:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.342097:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.342099:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.342100:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.342101:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.342103:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.342104:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.342105:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.342106:0:207610:0:(cl_object.c:351:cl_conf_set()) Process entered 00000080:00200000:2.0:1775773861.342107:0:207610:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b15:0x994:0x0]: losing layout lock 00000080:00000001:2.0:1775773861.342108:0:207610:0:(pcc.c:2213:pcc_layout_invalidate()) Process entered 00000080:00000001:2.0:1775773861.342109:0:207610:0:(pcc.c:2225:pcc_layout_invalidate()) Process leaving 00020000:00000001:2.0:1775773861.342110:0:207610:0:(lov_object.c:1412:lov_conf_set()) Process entered 00020000:00000001:2.0:1775773861.342110:0:207610:0:(lov_object.c:1426:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0) 00020000:00000002:2.0:1775773861.342112:0:207610:0:(lov_object.c:1499:lov_conf_set()) [0x200002b15:0x994:0x0] lo_layout_invalid=1 00020000:00000001:2.0:1775773861.342113:0:207610:0:(lov_object.c:1500:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342114:0:207610:0:(cl_object.c:359:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.342114:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.342115:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.342117:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.342118:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.342119:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.342121:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.342122:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.342123:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.342125:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.342126:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.342128:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.342129:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.342131:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.342145:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.342147:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.342149:0:207610:0:(file.c:6773:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.342150:0:207610:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:2.0:1775773861.342151:0:207610:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b15:0x994:0x0](ffff8f6226944b08) invalid 00000020:00000001:2.0:1775773861.342153:0:207610:0:(obd_sysfs.c:903:encode_fn_len()) Process filename at ffffaf0701cebca0 00000080:00002000:2.0:1775773861.342154:0:207610:0:(llite_internal.h:2024:d_lustre_invalidate()) invalidate dentry f.108 (ffff8f62094a0360) parent ffff8f6202184798 inode ffff8f6226944b08 refc 0 00000080:00000001:2.0:1775773861.342157:0:207610:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:2.0:1775773861.342160:0:207610:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:2.0:1775773861.342162:0:207610:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:2.0:1775773861.342164:0:207610:0:(namei.c:402:ll_lock_cancel_bits()) Process leaving 00000080:00000001:2.0:1775773861.342165:0:207610:0:(namei.c:536:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775773861.342166:0:207610:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1775773861.342167:0:207610:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1775773861.342168:0:207610:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1775773861.342169:0:207610:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1775773861.342171:0:207610:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775773861.342172:0:207610:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8f62265d6200 with handle 0xf74d1861c614846 from hash 00010000:00000001:2.0:1775773861.342174:0:207610:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1775773861.342175:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1775773861.342175:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.342176:0:207610:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1775773861.342177:0:207610:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1775773861.342178:0:207610:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:2.0:1775773861.342179:0:207610:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1775773861.342181:0:207610:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226129200/0xf74d1861c61483f lrc: 2/0,0 mode: PW/PW res: [0x200002b15:0x993:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x800028400000000 nid: local remote: 0xd88660ed39b2cddc expref: -99 pid: 214096 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00000001:2.0:1775773861.342185:0:207610:0:(namei.c:472:ll_md_blocking_ast()) Process entered 00000080:00000001:2.0:1775773861.342186:0:207610:0:(namei.c:239:ll_lock_cancel_bits()) Process entered 00000080:00000001:2.0:1775773861.342187:0:207610:0:(file.c:6026:ll_have_md_lock()) Process entered 00000080:00000040:2.0:1775773861.342188:0:207610:0:(file.c:6032:ll_have_md_lock()) trying to match res [0x200002b15:0x993:0x0] mode (null) 00000002:00000001:2.0:1775773861.342189:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.342190:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.342192:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960800 count: 2 00010000:00000001:2.0:1775773861.342193:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.342194:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342195:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.342197:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.342198:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960800 count: 2 00010000:00000001:2.0:1775773861.342199:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.342200:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342201:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.342202:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.342203:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960800 count: 2 00010000:00000001:2.0:1775773861.342204:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.342205:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342206:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.342207:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.342208:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960800 count: 2 00010000:00000001:2.0:1775773861.342209:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.342210:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342211:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.342211:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.342212:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960800 count: 2 00010000:00000001:2.0:1775773861.342213:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.342215:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.342216:0:207610:0:(file.c:6054:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.342216:0:207610:0:(namei.c:211:ll_dom_lock_cancel()) Process entered 00000020:00000001:2.0:1775773861.342217:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.342219:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.342220:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.342222:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.342223:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.342225:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.342226:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.342228:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.342229:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.342230:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.342232:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.342233:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.342235:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.342236:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.342238:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.342239:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.342241:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.342242:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00020000:00000002:2.0:1775773861.342243:0:207610:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff8f622e196c98) owner 0000000000000000/ffff8f62012d8000 00020000:00000001:2.0:1775773861.342244:0:207610:0:(lov_object.c:1110:lov_flush_composite()) Process entered 00000020:00000001:2.0:1775773861.342246:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00000002:00000001:2.0:1775773861.342247:0:207610:0:(mdc_dev.c:306:mdc_dlm_canceling()) Process entered 00000002:00000001:2.0:1775773861.342248:0:207610:0:(mdc_dev.c:255:mdc_lock_flush()) Process entered 00000008:00000001:2.0:1775773861.342249:0:207610:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775773861.342250:0:207610:0:(osc_cache.c:3200:osc_cache_wait_range()) Process entered 00000008:00000020:2.0:1775773861.342251:0:207610:0:(osc_cache.c:3237:osc_cache_wait_range()) obj ffff8f623476f6c0 ready 0|-|- wr 0|-|- rd 0|- sync file range. 00000008:00000001:2.0:1775773861.342253:0:207610:0:(osc_cache.c:3238:osc_cache_wait_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775773861.342254:0:207610:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff8f623476f6c0 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:2.0:1775773861.342256:0:207610:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000002:00000020:2.0:1775773861.342257:0:207610:0:(mdc_dev.c:262:mdc_lock_flush()) object ffff8f623476f6c0: [0 -> 18446744073709551615] 0 pages were written back. 00000002:00000001:2.0:1775773861.342259:0:207610:0:(mdc_dev.c:228:mdc_lock_discard_pages()) Process entered 00000020:00000001:2.0:1775773861.342260:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476f6c0 -> ffff8f6202fc66c0 00000020:00000001:2.0:1775773861.342261:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f6202fc66c0 -> ffff8f6202fc66c0 00000020:00000001:2.0:1775773861.342262:0:207610:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775773861.342262:0:207610:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775773861.342263:0:207610:0:(vvp_io.c:1817:vvp_io_init()) [0x200002b15:0x993:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775773861.342265:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.342265:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775773861.342266:0:207610:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.342267:0:207610:0:(lov_object.c:1554:lov_io_init()) [0x200002b15:0x993:0x0]io ffff8f6205260bc8 type 8 ignore/verify layout 1/0 00020000:00000001:2.0:1775773861.342269:0:207610:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775773861.342269:0:207610:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775773861.342271:0:207610:0:(lov_io.c:599:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775773861.342272:0:207610:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775773861.342273:0:207610:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342274:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.342274:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775773861.342275:0:207610:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000001:2.0:1775773861.342276:0:207610:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775773861.342277:0:207610:0:(osc_cache.c:3447:osc_page_gang_lookup()) Process entered 00000008:00000001:2.0:1775773861.342278:0:207610:0:(osc_cache.c:3524:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775773861.342280:0:207610:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775773861.342280:0:207610:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775773861.342281:0:207610:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775773861.342282:0:207610:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775773861.342283:0:207610:0:(vvp_io.c:288:vvp_io_fini()) [0x200002b15:0x993:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775773861.342285:0:207610:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775773861.342285:0:207610:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000002:00000001:2.0:1775773861.342286:0:207610:0:(mdc_dev.c:245:mdc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342287:0:207610:0:(mdc_dev.c:273:mdc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342288:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476f6c0 -> ffff8f6202fc66c0 00000020:00000001:2.0:1775773861.342289:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476f6c0 -> ffff8f6202fc66c0 00000020:00000001:2.0:1775773861.342290:0:207610:0:(cl_object.c:256:cl_object_attr_update()) Process entered 00000008:00000020:2.0:1775773861.342291:0:207610:0:(osc_object.c:161:osc_attr_update()) set kms from 131072to 0 00020000:00000001:2.0:1775773861.342292:0:207610:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:2.0:1775773861.342293:0:207610:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342294:0:207610:0:(cl_object.c:269:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342295:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476f6c0 -> ffff8f6202fc66c0 00000002:00000001:2.0:1775773861.342296:0:207610:0:(mdc_dev.c:346:mdc_dlm_canceling()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342297:0:207610:0:(mdc_dev.c:1535:mdc_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342298:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775773861.342299:0:207610:0:(lov_object.c:1119:lov_flush_composite()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.342300:0:207610:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff8f622e196c98) owner 0000000000000000/ffff8f62012d8000 00000020:00000001:2.0:1775773861.342302:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.342303:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.342304:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.342306:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.342307:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.342309:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.342310:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.342312:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.342313:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.342315:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.342316:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.342318:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.342319:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.342321:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.342323:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.342324:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.342326:0:207610:0:(namei.c:229:ll_dom_lock_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.342327:0:207610:0:(file.c:6731:ll_layout_conf()) Process entered 00000020:00000001:2.0:1775773861.342328:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.342329:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.342330:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.342331:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.342333:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.342334:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.342336:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.342337:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.342339:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.342340:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.342342:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.342343:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.342344:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.342346:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.342347:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.342348:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.342349:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.342350:0:207610:0:(cl_object.c:351:cl_conf_set()) Process entered 00000080:00200000:2.0:1775773861.342351:0:207610:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b15:0x993:0x0]: losing layout lock 00000080:00000001:2.0:1775773861.342352:0:207610:0:(pcc.c:2213:pcc_layout_invalidate()) Process entered 00000080:00000001:2.0:1775773861.342353:0:207610:0:(pcc.c:2225:pcc_layout_invalidate()) Process leaving 00020000:00000001:2.0:1775773861.342354:0:207610:0:(lov_object.c:1412:lov_conf_set()) Process entered 00020000:00000001:2.0:1775773861.342355:0:207610:0:(lov_object.c:1426:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0) 00020000:00000002:2.0:1775773861.342356:0:207610:0:(lov_object.c:1499:lov_conf_set()) [0x200002b15:0x993:0x0] lo_layout_invalid=1 00020000:00000001:2.0:1775773861.342357:0:207610:0:(lov_object.c:1500:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342358:0:207610:0:(cl_object.c:359:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.342359:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.342360:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.342361:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.342362:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.342364:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.342365:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.342367:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.342368:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.342369:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.342371:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.342372:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.342373:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.342375:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.342376:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.342377:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.342378:0:207610:0:(file.c:6773:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.342379:0:207610:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:2.0:1775773861.342380:0:207610:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b15:0x993:0x0](ffff8f621efba1c8) invalid 00000020:00000001:2.0:1775773861.342382:0:207610:0:(obd_sysfs.c:903:encode_fn_len()) Process filename at ffffaf0701cebca0 00000080:00002000:2.0:1775773861.342383:0:207610:0:(llite_internal.h:2024:d_lustre_invalidate()) invalidate dentry f.107 (ffff8f62186ac5e8) parent ffff8f6202184798 inode ffff8f621efba1c8 refc 0 00000080:00000001:2.0:1775773861.342385:0:207610:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:2.0:1775773861.342387:0:207610:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:2.0:1775773861.342389:0:207610:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:2.0:1775773861.342390:0:207610:0:(namei.c:402:ll_lock_cancel_bits()) Process leaving 00000080:00000001:2.0:1775773861.342391:0:207610:0:(namei.c:536:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775773861.342392:0:207610:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1775773861.342393:0:207610:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1775773861.342394:0:207610:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1775773861.342395:0:207610:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1775773861.342396:0:207610:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775773861.342398:0:207610:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8f6226129200 with handle 0xf74d1861c61483f from hash 00010000:00000001:2.0:1775773861.342399:0:207610:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1775773861.342399:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1775773861.342400:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.342401:0:207610:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1775773861.342402:0:207610:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1775773861.342403:0:207610:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:2.0:1775773861.342405:0:207610:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1775773861.342407:0:207610:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f621839f800/0xf74d1861c614838 lrc: 2/0,0 mode: PW/PW res: [0x200002b15:0x992:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x800028400000000 nid: local remote: 0xd88660ed39b2cdb9 expref: -99 pid: 214095 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00000001:2.0:1775773861.342412:0:207610:0:(namei.c:472:ll_md_blocking_ast()) Process entered 00000080:00000001:2.0:1775773861.342413:0:207610:0:(namei.c:239:ll_lock_cancel_bits()) Process entered 00000080:00000001:2.0:1775773861.342414:0:207610:0:(file.c:6026:ll_have_md_lock()) Process entered 00000080:00000040:2.0:1775773861.342415:0:207610:0:(file.c:6032:ll_have_md_lock()) trying to match res [0x200002b15:0x992:0x0] mode (null) 00000002:00000001:2.0:1775773861.342416:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.342417:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.342418:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6202d82100 count: 2 00010000:00000001:2.0:1775773861.342419:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.342421:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342422:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.342422:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.342424:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6202d82100 count: 2 00010000:00000001:2.0:1775773861.342425:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.342426:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342427:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.342428:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.342429:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6202d82100 count: 2 00010000:00000001:2.0:1775773861.342430:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.342431:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342432:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.342432:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.342433:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6202d82100 count: 2 00010000:00000001:2.0:1775773861.342434:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.342435:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342436:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.342437:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.342438:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6202d82100 count: 2 00010000:00000001:2.0:1775773861.342439:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.342440:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.342441:0:207610:0:(file.c:6054:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.342442:0:207610:0:(namei.c:211:ll_dom_lock_cancel()) Process entered 00000020:00000001:2.0:1775773861.342442:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.342444:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.342445:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.342447:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.342448:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.342450:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.342451:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.342453:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.342454:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.342456:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.342457:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.342458:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.342460:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.342461:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.342463:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.342464:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.342465:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.342466:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00020000:00000002:2.0:1775773861.342468:0:207610:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff8f621029f5d0) owner 0000000000000000/ffff8f62012d8000 00020000:00000001:2.0:1775773861.342482:0:207610:0:(lov_object.c:1110:lov_flush_composite()) Process entered 00000020:00000001:2.0:1775773861.342483:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00000002:00000001:2.0:1775773861.342484:0:207610:0:(mdc_dev.c:306:mdc_dlm_canceling()) Process entered 00000002:00000001:2.0:1775773861.342485:0:207610:0:(mdc_dev.c:255:mdc_lock_flush()) Process entered 00000008:00000001:2.0:1775773861.342486:0:207610:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775773861.342487:0:207610:0:(osc_cache.c:3200:osc_cache_wait_range()) Process entered 00000008:00000020:2.0:1775773861.342488:0:207610:0:(osc_cache.c:3237:osc_cache_wait_range()) obj ffff8f623476fdc0 ready 0|-|- wr 0|-|- rd 0|- sync file range. 00000008:00000001:2.0:1775773861.342490:0:207610:0:(osc_cache.c:3238:osc_cache_wait_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775773861.342492:0:207610:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff8f623476fdc0 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:2.0:1775773861.342494:0:207610:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000002:00000020:2.0:1775773861.342495:0:207610:0:(mdc_dev.c:262:mdc_lock_flush()) object ffff8f623476fdc0: [0 -> 18446744073709551615] 0 pages were written back. 00000002:00000001:2.0:1775773861.342496:0:207610:0:(mdc_dev.c:228:mdc_lock_discard_pages()) Process entered 00000020:00000001:2.0:1775773861.342497:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476fdc0 -> ffff8f622d9ae9e0 00000020:00000001:2.0:1775773861.342498:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f622d9ae9e0 -> ffff8f622d9ae9e0 00000020:00000001:2.0:1775773861.342499:0:207610:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775773861.342500:0:207610:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775773861.342501:0:207610:0:(vvp_io.c:1817:vvp_io_init()) [0x200002b15:0x992:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775773861.342502:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.342503:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775773861.342504:0:207610:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.342505:0:207610:0:(lov_object.c:1554:lov_io_init()) [0x200002b15:0x992:0x0]io ffff8f6205260bc8 type 8 ignore/verify layout 1/0 00020000:00000001:2.0:1775773861.342507:0:207610:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775773861.342508:0:207610:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775773861.342509:0:207610:0:(lov_io.c:599:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775773861.342510:0:207610:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775773861.342511:0:207610:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342512:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.342512:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775773861.342513:0:207610:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000001:2.0:1775773861.342514:0:207610:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775773861.342515:0:207610:0:(osc_cache.c:3447:osc_page_gang_lookup()) Process entered 00000008:00000001:2.0:1775773861.342516:0:207610:0:(osc_cache.c:3524:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775773861.342517:0:207610:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775773861.342517:0:207610:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775773861.342518:0:207610:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775773861.342519:0:207610:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775773861.342520:0:207610:0:(vvp_io.c:288:vvp_io_fini()) [0x200002b15:0x992:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775773861.342521:0:207610:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775773861.342522:0:207610:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000002:00000001:2.0:1775773861.342523:0:207610:0:(mdc_dev.c:245:mdc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342524:0:207610:0:(mdc_dev.c:273:mdc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342525:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476fdc0 -> ffff8f622d9ae9e0 00000020:00000001:2.0:1775773861.342526:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476fdc0 -> ffff8f622d9ae9e0 00000020:00000001:2.0:1775773861.342527:0:207610:0:(cl_object.c:256:cl_object_attr_update()) Process entered 00000008:00000020:2.0:1775773861.342528:0:207610:0:(osc_object.c:161:osc_attr_update()) set kms from 131072to 0 00020000:00000001:2.0:1775773861.342529:0:207610:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:2.0:1775773861.342530:0:207610:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342531:0:207610:0:(cl_object.c:269:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342532:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476fdc0 -> ffff8f622d9ae9e0 00000002:00000001:2.0:1775773861.342534:0:207610:0:(mdc_dev.c:346:mdc_dlm_canceling()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342535:0:207610:0:(mdc_dev.c:1535:mdc_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342536:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775773861.342537:0:207610:0:(lov_object.c:1119:lov_flush_composite()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.342538:0:207610:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff8f621029f5d0) owner 0000000000000000/ffff8f62012d8000 00000020:00000001:2.0:1775773861.342539:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.342540:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.342542:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.342544:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.342546:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.342547:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.342549:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.342550:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.342552:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.342553:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.342555:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.342556:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.342558:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.342559:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.342561:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.342562:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.342564:0:207610:0:(namei.c:229:ll_dom_lock_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.342565:0:207610:0:(file.c:6731:ll_layout_conf()) Process entered 00000020:00000001:2.0:1775773861.342566:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.342567:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.342569:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.342570:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.342572:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.342573:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.342575:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.342576:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.342578:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.342579:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.342580:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.342582:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.342583:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.342585:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.342586:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.342588:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.342589:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.342590:0:207610:0:(cl_object.c:351:cl_conf_set()) Process entered 00000080:00200000:2.0:1775773861.342591:0:207610:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b15:0x992:0x0]: losing layout lock 00000080:00000001:2.0:1775773861.342592:0:207610:0:(pcc.c:2213:pcc_layout_invalidate()) Process entered 00000080:00000001:2.0:1775773861.342593:0:207610:0:(pcc.c:2225:pcc_layout_invalidate()) Process leaving 00020000:00000001:2.0:1775773861.342594:0:207610:0:(lov_object.c:1412:lov_conf_set()) Process entered 00020000:00000001:2.0:1775773861.342595:0:207610:0:(lov_object.c:1426:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0) 00020000:00000002:2.0:1775773861.342597:0:207610:0:(lov_object.c:1499:lov_conf_set()) [0x200002b15:0x992:0x0] lo_layout_invalid=1 00020000:00000001:2.0:1775773861.342598:0:207610:0:(lov_object.c:1500:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342599:0:207610:0:(cl_object.c:359:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.342600:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.342601:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.342603:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.342604:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.342605:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.342607:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.342608:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.342609:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.342611:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.342612:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.342613:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.342614:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.342616:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.342617:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.342618:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.342619:0:207610:0:(file.c:6773:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.342620:0:207610:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:2.0:1775773861.342621:0:207610:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b15:0x992:0x0](ffff8f621efb3248) invalid 00000020:00000001:2.0:1775773861.342623:0:207610:0:(obd_sysfs.c:903:encode_fn_len()) Process filename at ffffaf0701cebca0 00000080:00002000:2.0:1775773861.342624:0:207610:0:(llite_internal.h:2024:d_lustre_invalidate()) invalidate dentry f.106 (ffff8f62191c5360) parent ffff8f6202184798 inode ffff8f621efb3248 refc 0 00000080:00000001:2.0:1775773861.342627:0:207610:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:2.0:1775773861.342630:0:207610:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:2.0:1775773861.342632:0:207610:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:2.0:1775773861.342633:0:207610:0:(namei.c:402:ll_lock_cancel_bits()) Process leaving 00000080:00000001:2.0:1775773861.342634:0:207610:0:(namei.c:536:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775773861.342635:0:207610:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1775773861.342636:0:207610:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1775773861.342637:0:207610:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1775773861.342638:0:207610:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1775773861.342639:0:207610:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775773861.342641:0:207610:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8f621839f800 with handle 0xf74d1861c614838 from hash 00010000:00000001:2.0:1775773861.342643:0:207610:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1775773861.342643:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1775773861.342644:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.342645:0:207610:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1775773861.342646:0:207610:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1775773861.342647:0:207610:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:2.0:1775773861.342648:0:207610:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1775773861.342650:0:207610:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226128400/0xf74d1861c614831 lrc: 2/0,0 mode: PW/PW res: [0x200002b15:0x991:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x800028400000000 nid: local remote: 0xd88660ed39b2cd96 expref: -99 pid: 214094 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00000001:2.0:1775773861.342654:0:207610:0:(namei.c:472:ll_md_blocking_ast()) Process entered 00000080:00000001:2.0:1775773861.342655:0:207610:0:(namei.c:239:ll_lock_cancel_bits()) Process entered 00000080:00000001:2.0:1775773861.342656:0:207610:0:(file.c:6026:ll_have_md_lock()) Process entered 00000080:00000040:2.0:1775773861.342657:0:207610:0:(file.c:6032:ll_have_md_lock()) trying to match res [0x200002b15:0x991:0x0] mode (null) 00000002:00000001:2.0:1775773861.342658:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.342659:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.342661:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960e00 count: 2 00010000:00000001:2.0:1775773861.342662:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.342663:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342664:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.342665:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.342666:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960e00 count: 2 00010000:00000001:2.0:1775773861.342668:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.342669:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342670:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.342671:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.342672:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960e00 count: 2 00010000:00000001:2.0:1775773861.342673:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.342674:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342675:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.342676:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.342676:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960e00 count: 2 00010000:00000001:2.0:1775773861.342678:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.342679:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342680:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.342681:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.342682:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960e00 count: 2 00010000:00000001:2.0:1775773861.342683:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.342684:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.342685:0:207610:0:(file.c:6054:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.342686:0:207610:0:(namei.c:211:ll_dom_lock_cancel()) Process entered 00000020:00000001:2.0:1775773861.342687:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.342688:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.342690:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.342691:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.342692:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.342694:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.342695:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.342696:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.342698:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.342699:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.342701:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.342702:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.342703:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.342705:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.342706:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.342708:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.342709:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.342710:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00020000:00000002:2.0:1775773861.342711:0:207610:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff8f622e196000) owner 0000000000000000/ffff8f62012d8000 00020000:00000001:2.0:1775773861.342712:0:207610:0:(lov_object.c:1110:lov_flush_composite()) Process entered 00000020:00000001:2.0:1775773861.342714:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00000002:00000001:2.0:1775773861.342715:0:207610:0:(mdc_dev.c:306:mdc_dlm_canceling()) Process entered 00000002:00000001:2.0:1775773861.342716:0:207610:0:(mdc_dev.c:255:mdc_lock_flush()) Process entered 00000008:00000001:2.0:1775773861.342716:0:207610:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775773861.342717:0:207610:0:(osc_cache.c:3200:osc_cache_wait_range()) Process entered 00000008:00000020:2.0:1775773861.342718:0:207610:0:(osc_cache.c:3237:osc_cache_wait_range()) obj ffff8f623476efc0 ready 0|-|- wr 0|-|- rd 0|- sync file range. 00000008:00000001:2.0:1775773861.342720:0:207610:0:(osc_cache.c:3238:osc_cache_wait_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775773861.342722:0:207610:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff8f623476efc0 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:2.0:1775773861.342724:0:207610:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000002:00000020:2.0:1775773861.342725:0:207610:0:(mdc_dev.c:262:mdc_lock_flush()) object ffff8f623476efc0: [0 -> 18446744073709551615] 0 pages were written back. 00000002:00000001:2.0:1775773861.342727:0:207610:0:(mdc_dev.c:228:mdc_lock_discard_pages()) Process entered 00000020:00000001:2.0:1775773861.342728:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476efc0 -> ffff8f622c2e0788 00000020:00000001:2.0:1775773861.342729:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f622c2e0788 -> ffff8f622c2e0788 00000020:00000001:2.0:1775773861.342729:0:207610:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775773861.342730:0:207610:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775773861.342731:0:207610:0:(vvp_io.c:1817:vvp_io_init()) [0x200002b15:0x991:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775773861.342733:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.342733:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775773861.342734:0:207610:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.342735:0:207610:0:(lov_object.c:1554:lov_io_init()) [0x200002b15:0x991:0x0]io ffff8f6205260bc8 type 8 ignore/verify layout 1/0 00020000:00000001:2.0:1775773861.342737:0:207610:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775773861.342737:0:207610:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775773861.342738:0:207610:0:(lov_io.c:599:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775773861.342739:0:207610:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775773861.342740:0:207610:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342741:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.342742:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775773861.342742:0:207610:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000001:2.0:1775773861.342743:0:207610:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775773861.342744:0:207610:0:(osc_cache.c:3447:osc_page_gang_lookup()) Process entered 00000008:00000001:2.0:1775773861.342745:0:207610:0:(osc_cache.c:3524:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775773861.342746:0:207610:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775773861.342747:0:207610:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775773861.342748:0:207610:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775773861.342748:0:207610:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775773861.342750:0:207610:0:(vvp_io.c:288:vvp_io_fini()) [0x200002b15:0x991:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775773861.342751:0:207610:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775773861.342752:0:207610:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000002:00000001:2.0:1775773861.342753:0:207610:0:(mdc_dev.c:245:mdc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342754:0:207610:0:(mdc_dev.c:273:mdc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342755:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476efc0 -> ffff8f622c2e0788 00000020:00000001:2.0:1775773861.342756:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476efc0 -> ffff8f622c2e0788 00000020:00000001:2.0:1775773861.342757:0:207610:0:(cl_object.c:256:cl_object_attr_update()) Process entered 00000008:00000020:2.0:1775773861.342758:0:207610:0:(osc_object.c:161:osc_attr_update()) set kms from 131072to 0 00020000:00000001:2.0:1775773861.342759:0:207610:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:2.0:1775773861.342760:0:207610:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342762:0:207610:0:(cl_object.c:269:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342763:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476efc0 -> ffff8f622c2e0788 00000002:00000001:2.0:1775773861.342764:0:207610:0:(mdc_dev.c:346:mdc_dlm_canceling()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342765:0:207610:0:(mdc_dev.c:1535:mdc_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342766:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775773861.342767:0:207610:0:(lov_object.c:1119:lov_flush_composite()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.342768:0:207610:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff8f622e196000) owner 0000000000000000/ffff8f62012d8000 00000020:00000001:2.0:1775773861.342769:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.342770:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.342771:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.342773:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.342775:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.342776:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.342777:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.342779:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.342781:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.342782:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.342784:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.342785:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.342786:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.342788:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.342789:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.342790:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.342791:0:207610:0:(namei.c:229:ll_dom_lock_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.342792:0:207610:0:(file.c:6731:ll_layout_conf()) Process entered 00000020:00000001:2.0:1775773861.342793:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.342794:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.342796:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.342797:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.342798:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.342800:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.342801:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.342803:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.342805:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.342806:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.342808:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.342809:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.342810:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.342812:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.342813:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.342814:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.342815:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.342816:0:207610:0:(cl_object.c:351:cl_conf_set()) Process entered 00000080:00200000:2.0:1775773861.342817:0:207610:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b15:0x991:0x0]: losing layout lock 00000080:00000001:2.0:1775773861.342818:0:207610:0:(pcc.c:2213:pcc_layout_invalidate()) Process entered 00000080:00000001:2.0:1775773861.342819:0:207610:0:(pcc.c:2225:pcc_layout_invalidate()) Process leaving 00020000:00000001:2.0:1775773861.342820:0:207610:0:(lov_object.c:1412:lov_conf_set()) Process entered 00020000:00000001:2.0:1775773861.342821:0:207610:0:(lov_object.c:1426:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0) 00020000:00000002:2.0:1775773861.342822:0:207610:0:(lov_object.c:1499:lov_conf_set()) [0x200002b15:0x991:0x0] lo_layout_invalid=1 00020000:00000001:2.0:1775773861.342823:0:207610:0:(lov_object.c:1500:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342824:0:207610:0:(cl_object.c:359:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.342825:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.342826:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.342827:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.342828:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.342830:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.342831:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.342832:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.342834:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.342835:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.342836:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.342838:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.342839:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.342841:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.342842:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.342843:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.342845:0:207610:0:(file.c:6773:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.342846:0:207610:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:2.0:1775773861.342847:0:207610:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b15:0x991:0x0](ffff8f621efb9988) invalid 00000020:00000001:2.0:1775773861.342849:0:207610:0:(obd_sysfs.c:903:encode_fn_len()) Process filename at ffffaf0701cebca0 00000080:00002000:2.0:1775773861.342850:0:207610:0:(llite_internal.h:2024:d_lustre_invalidate()) invalidate dentry f.105 (ffff8f621926c510) parent ffff8f6202184798 inode ffff8f621efb9988 refc 0 00000080:00000001:2.0:1775773861.342853:0:207610:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:2.0:1775773861.342855:0:207610:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:2.0:1775773861.342858:0:207610:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:2.0:1775773861.342859:0:207610:0:(namei.c:402:ll_lock_cancel_bits()) Process leaving 00000080:00000001:2.0:1775773861.342860:0:207610:0:(namei.c:536:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775773861.342861:0:207610:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1775773861.342862:0:207610:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1775773861.342863:0:207610:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1775773861.342864:0:207610:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1775773861.342866:0:207610:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775773861.342867:0:207610:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8f6226128400 with handle 0xf74d1861c614831 from hash 00010000:00000001:2.0:1775773861.342868:0:207610:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1775773861.342869:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1775773861.342870:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.342871:0:207610:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1775773861.342872:0:207610:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1775773861.342873:0:207610:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:2.0:1775773861.342874:0:207610:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1775773861.342876:0:207610:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476dc00/0xf74d1861c61482a lrc: 2/0,0 mode: PW/PW res: [0x200002b15:0x990:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x800028400000000 nid: local remote: 0xd88660ed39b2cd73 expref: -99 pid: 214093 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00000001:2.0:1775773861.342880:0:207610:0:(namei.c:472:ll_md_blocking_ast()) Process entered 00000080:00000001:2.0:1775773861.342881:0:207610:0:(namei.c:239:ll_lock_cancel_bits()) Process entered 00000080:00000001:2.0:1775773861.342882:0:207610:0:(file.c:6026:ll_have_md_lock()) Process entered 00000080:00000040:2.0:1775773861.342883:0:207610:0:(file.c:6032:ll_have_md_lock()) trying to match res [0x200002b15:0x990:0x0] mode (null) 00000002:00000001:2.0:1775773861.342884:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.342885:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.342886:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6234adc800 count: 2 00010000:00000001:2.0:1775773861.342888:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.342889:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342890:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.342891:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.342892:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6234adc800 count: 2 00010000:00000001:2.0:1775773861.342894:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.342895:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342896:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.342897:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.342898:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6234adc800 count: 2 00010000:00000001:2.0:1775773861.342899:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.342900:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342901:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.342901:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.342902:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6234adc800 count: 2 00010000:00000001:2.0:1775773861.342903:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.342904:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342905:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.342906:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.342907:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6234adc800 count: 2 00010000:00000001:2.0:1775773861.342908:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.342909:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.342910:0:207610:0:(file.c:6054:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.342911:0:207610:0:(namei.c:211:ll_dom_lock_cancel()) Process entered 00000020:00000001:2.0:1775773861.342912:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.342913:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.342914:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.342916:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.342917:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.342918:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.342920:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.342921:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.342923:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.342924:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.342925:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.342927:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.342928:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.342929:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.342931:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.342932:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.342934:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.342935:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00020000:00000002:2.0:1775773861.342937:0:207610:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff8f62057942e8) owner 0000000000000000/ffff8f62012d8000 00020000:00000001:2.0:1775773861.342938:0:207610:0:(lov_object.c:1110:lov_flush_composite()) Process entered 00000020:00000001:2.0:1775773861.342939:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00000002:00000001:2.0:1775773861.342940:0:207610:0:(mdc_dev.c:306:mdc_dlm_canceling()) Process entered 00000002:00000001:2.0:1775773861.342941:0:207610:0:(mdc_dev.c:255:mdc_lock_flush()) Process entered 00000008:00000001:2.0:1775773861.342942:0:207610:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775773861.342942:0:207610:0:(osc_cache.c:3200:osc_cache_wait_range()) Process entered 00000008:00000020:2.0:1775773861.342944:0:207610:0:(osc_cache.c:3237:osc_cache_wait_range()) obj ffff8f622d9ac380 ready 0|-|- wr 0|-|- rd 0|- sync file range. 00000008:00000001:2.0:1775773861.342945:0:207610:0:(osc_cache.c:3238:osc_cache_wait_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775773861.342947:0:207610:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff8f622d9ac380 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:2.0:1775773861.342949:0:207610:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000002:00000020:2.0:1775773861.342950:0:207610:0:(mdc_dev.c:262:mdc_lock_flush()) object ffff8f622d9ac380: [0 -> 18446744073709551615] 0 pages were written back. 00000002:00000001:2.0:1775773861.342951:0:207610:0:(mdc_dev.c:228:mdc_lock_discard_pages()) Process entered 00000020:00000001:2.0:1775773861.342952:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f622d9ac380 -> ffff8f622612ad00 00000020:00000001:2.0:1775773861.342953:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f622612ad00 -> ffff8f622612ad00 00000020:00000001:2.0:1775773861.342954:0:207610:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775773861.342955:0:207610:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775773861.342956:0:207610:0:(vvp_io.c:1817:vvp_io_init()) [0x200002b15:0x990:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775773861.342957:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.342958:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775773861.342958:0:207610:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.342960:0:207610:0:(lov_object.c:1554:lov_io_init()) [0x200002b15:0x990:0x0]io ffff8f6205260bc8 type 8 ignore/verify layout 1/0 00020000:00000001:2.0:1775773861.342961:0:207610:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775773861.342962:0:207610:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775773861.342963:0:207610:0:(lov_io.c:599:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775773861.342964:0:207610:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775773861.342964:0:207610:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342965:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.342966:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775773861.342967:0:207610:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000001:2.0:1775773861.342967:0:207610:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775773861.342968:0:207610:0:(osc_cache.c:3447:osc_page_gang_lookup()) Process entered 00000008:00000001:2.0:1775773861.342970:0:207610:0:(osc_cache.c:3524:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775773861.342971:0:207610:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775773861.342972:0:207610:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775773861.342972:0:207610:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775773861.342973:0:207610:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775773861.342975:0:207610:0:(vvp_io.c:288:vvp_io_fini()) [0x200002b15:0x990:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775773861.342976:0:207610:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775773861.342977:0:207610:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000002:00000001:2.0:1775773861.342977:0:207610:0:(mdc_dev.c:245:mdc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342978:0:207610:0:(mdc_dev.c:273:mdc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342979:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f622d9ac380 -> ffff8f622612ad00 00000020:00000001:2.0:1775773861.342980:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f622d9ac380 -> ffff8f622612ad00 00000020:00000001:2.0:1775773861.342981:0:207610:0:(cl_object.c:256:cl_object_attr_update()) Process entered 00000008:00000020:2.0:1775773861.342982:0:207610:0:(osc_object.c:161:osc_attr_update()) set kms from 131072to 0 00020000:00000001:2.0:1775773861.342983:0:207610:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:2.0:1775773861.342984:0:207610:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342985:0:207610:0:(cl_object.c:269:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342986:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f622d9ac380 -> ffff8f622612ad00 00000002:00000001:2.0:1775773861.342987:0:207610:0:(mdc_dev.c:346:mdc_dlm_canceling()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.342988:0:207610:0:(mdc_dev.c:1535:mdc_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.342990:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775773861.342991:0:207610:0:(lov_object.c:1119:lov_flush_composite()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.342992:0:207610:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff8f62057942e8) owner 0000000000000000/ffff8f62012d8000 00000020:00000001:2.0:1775773861.342993:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.342994:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.342995:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.342996:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.342998:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.342999:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.343000:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.343002:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.343003:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.343005:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.343006:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.343008:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.343009:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.343011:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.343013:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.343014:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.343016:0:207610:0:(namei.c:229:ll_dom_lock_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.343017:0:207610:0:(file.c:6731:ll_layout_conf()) Process entered 00000020:00000001:2.0:1775773861.343018:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.343019:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.343020:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.343021:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.343023:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.343024:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.343025:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.343027:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.343028:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.343029:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.343031:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.343032:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.343033:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.343035:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.343036:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.343038:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.343039:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.343040:0:207610:0:(cl_object.c:351:cl_conf_set()) Process entered 00000080:00200000:2.0:1775773861.343041:0:207610:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b15:0x990:0x0]: losing layout lock 00000080:00000001:2.0:1775773861.343042:0:207610:0:(pcc.c:2213:pcc_layout_invalidate()) Process entered 00000080:00000001:2.0:1775773861.343042:0:207610:0:(pcc.c:2225:pcc_layout_invalidate()) Process leaving 00020000:00000001:2.0:1775773861.343043:0:207610:0:(lov_object.c:1412:lov_conf_set()) Process entered 00020000:00000001:2.0:1775773861.343044:0:207610:0:(lov_object.c:1426:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0) 00020000:00000002:2.0:1775773861.343045:0:207610:0:(lov_object.c:1499:lov_conf_set()) [0x200002b15:0x990:0x0] lo_layout_invalid=1 00020000:00000001:2.0:1775773861.343046:0:207610:0:(lov_object.c:1500:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343047:0:207610:0:(cl_object.c:359:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.343048:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.343049:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.343051:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.343052:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.343054:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.343055:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.343056:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.343058:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.343059:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.343060:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.343062:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.343063:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.343064:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.343066:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.343067:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.343068:0:207610:0:(file.c:6773:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.343069:0:207610:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:2.0:1775773861.343070:0:207610:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b15:0x990:0x0](ffff8f6226942a08) invalid 00000020:00000001:2.0:1775773861.343071:0:207610:0:(obd_sysfs.c:903:encode_fn_len()) Process filename at ffffaf0701cebca0 00000080:00002000:2.0:1775773861.343072:0:207610:0:(llite_internal.h:2024:d_lustre_invalidate()) invalidate dentry f.104 (ffff8f6209b2dd80) parent ffff8f6202184798 inode ffff8f6226942a08 refc 0 00000080:00000001:2.0:1775773861.343075:0:207610:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:2.0:1775773861.343078:0:207610:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:2.0:1775773861.343081:0:207610:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:2.0:1775773861.343082:0:207610:0:(namei.c:402:ll_lock_cancel_bits()) Process leaving 00000080:00000001:2.0:1775773861.343083:0:207610:0:(namei.c:536:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775773861.343084:0:207610:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1775773861.343085:0:207610:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1775773861.343086:0:207610:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1775773861.343087:0:207610:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1775773861.343088:0:207610:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775773861.343089:0:207610:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8f623476dc00 with handle 0xf74d1861c61482a from hash 00010000:00000001:2.0:1775773861.343090:0:207610:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1775773861.343091:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1775773861.343092:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.343093:0:207610:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1775773861.343094:0:207610:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1775773861.343095:0:207610:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:2.0:1775773861.343097:0:207610:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1775773861.343098:0:207610:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226128800/0xf74d1861c614823 lrc: 2/0,0 mode: PW/PW res: [0x200002b15:0x98f:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x800028400000000 nid: local remote: 0xd88660ed39b2cd50 expref: -99 pid: 214092 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00000001:2.0:1775773861.343103:0:207610:0:(namei.c:472:ll_md_blocking_ast()) Process entered 00000080:00000001:2.0:1775773861.343104:0:207610:0:(namei.c:239:ll_lock_cancel_bits()) Process entered 00000080:00000001:2.0:1775773861.343105:0:207610:0:(file.c:6026:ll_have_md_lock()) Process entered 00000080:00000040:2.0:1775773861.343106:0:207610:0:(file.c:6032:ll_have_md_lock()) trying to match res [0x200002b15:0x98f:0x0] mode (null) 00000002:00000001:2.0:1775773861.343107:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.343108:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.343109:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960900 count: 2 00010000:00000001:2.0:1775773861.343110:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.343112:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343112:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.343113:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.343114:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960900 count: 2 00010000:00000001:2.0:1775773861.343115:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.343117:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343118:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.343119:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.343119:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960900 count: 2 00010000:00000001:2.0:1775773861.343121:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.343122:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343123:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.343123:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.343124:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960900 count: 2 00010000:00000001:2.0:1775773861.343125:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.343126:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343127:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.343128:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.343129:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960900 count: 2 00010000:00000001:2.0:1775773861.343130:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.343131:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.343132:0:207610:0:(file.c:6054:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.343147:0:207610:0:(namei.c:211:ll_dom_lock_cancel()) Process entered 00000020:00000001:2.0:1775773861.343147:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.343149:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.343151:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.343153:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.343155:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.343156:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.343158:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.343159:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.343161:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.343163:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.343164:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.343166:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.343167:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.343169:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.343170:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.343172:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.343173:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.343174:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00020000:00000002:2.0:1775773861.343175:0:207610:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff8f622e1968b8) owner 0000000000000000/ffff8f62012d8000 00020000:00000001:2.0:1775773861.343177:0:207610:0:(lov_object.c:1110:lov_flush_composite()) Process entered 00000020:00000001:2.0:1775773861.343178:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00000002:00000001:2.0:1775773861.343179:0:207610:0:(mdc_dev.c:306:mdc_dlm_canceling()) Process entered 00000002:00000001:2.0:1775773861.343180:0:207610:0:(mdc_dev.c:255:mdc_lock_flush()) Process entered 00000008:00000001:2.0:1775773861.343181:0:207610:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775773861.343182:0:207610:0:(osc_cache.c:3200:osc_cache_wait_range()) Process entered 00000008:00000020:2.0:1775773861.343184:0:207610:0:(osc_cache.c:3237:osc_cache_wait_range()) obj ffff8f623476e700 ready 0|-|- wr 0|-|- rd 0|- sync file range. 00000008:00000001:2.0:1775773861.343185:0:207610:0:(osc_cache.c:3238:osc_cache_wait_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775773861.343187:0:207610:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff8f623476e700 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:2.0:1775773861.343189:0:207610:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000002:00000020:2.0:1775773861.343190:0:207610:0:(mdc_dev.c:262:mdc_lock_flush()) object ffff8f623476e700: [0 -> 18446744073709551615] 0 pages were written back. 00000002:00000001:2.0:1775773861.343192:0:207610:0:(mdc_dev.c:228:mdc_lock_discard_pages()) Process entered 00000020:00000001:2.0:1775773861.343192:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476e700 -> ffff8f622c2e0148 00000020:00000001:2.0:1775773861.343193:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f622c2e0148 -> ffff8f622c2e0148 00000020:00000001:2.0:1775773861.343194:0:207610:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775773861.343195:0:207610:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775773861.343196:0:207610:0:(vvp_io.c:1817:vvp_io_init()) [0x200002b15:0x98f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775773861.343197:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.343199:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775773861.343199:0:207610:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.343201:0:207610:0:(lov_object.c:1554:lov_io_init()) [0x200002b15:0x98f:0x0]io ffff8f6205260bc8 type 8 ignore/verify layout 1/0 00020000:00000001:2.0:1775773861.343202:0:207610:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775773861.343203:0:207610:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775773861.343204:0:207610:0:(lov_io.c:599:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775773861.343205:0:207610:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775773861.343206:0:207610:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343207:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.343207:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775773861.343208:0:207610:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000001:2.0:1775773861.343209:0:207610:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775773861.343210:0:207610:0:(osc_cache.c:3447:osc_page_gang_lookup()) Process entered 00000008:00000001:2.0:1775773861.343211:0:207610:0:(osc_cache.c:3524:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775773861.343212:0:207610:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775773861.343213:0:207610:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775773861.343213:0:207610:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775773861.343214:0:207610:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775773861.343215:0:207610:0:(vvp_io.c:288:vvp_io_fini()) [0x200002b15:0x98f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775773861.343216:0:207610:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775773861.343217:0:207610:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000002:00000001:2.0:1775773861.343218:0:207610:0:(mdc_dev.c:245:mdc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343219:0:207610:0:(mdc_dev.c:273:mdc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343220:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476e700 -> ffff8f622c2e0148 00000020:00000001:2.0:1775773861.343221:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476e700 -> ffff8f622c2e0148 00000020:00000001:2.0:1775773861.343222:0:207610:0:(cl_object.c:256:cl_object_attr_update()) Process entered 00000008:00000020:2.0:1775773861.343223:0:207610:0:(osc_object.c:161:osc_attr_update()) set kms from 131072to 0 00020000:00000001:2.0:1775773861.343224:0:207610:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:2.0:1775773861.343225:0:207610:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343226:0:207610:0:(cl_object.c:269:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343227:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476e700 -> ffff8f622c2e0148 00000002:00000001:2.0:1775773861.343229:0:207610:0:(mdc_dev.c:346:mdc_dlm_canceling()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343230:0:207610:0:(mdc_dev.c:1535:mdc_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343231:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775773861.343232:0:207610:0:(lov_object.c:1119:lov_flush_composite()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.343233:0:207610:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff8f622e1968b8) owner 0000000000000000/ffff8f62012d8000 00000020:00000001:2.0:1775773861.343234:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.343236:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.343237:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.343239:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.343241:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.343242:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.343244:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.343245:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.343247:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.343249:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.343250:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.343251:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.343253:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.343255:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.343256:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.343257:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.343258:0:207610:0:(namei.c:229:ll_dom_lock_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.343259:0:207610:0:(file.c:6731:ll_layout_conf()) Process entered 00000020:00000001:2.0:1775773861.343260:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.343261:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.343263:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.343264:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.343265:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.343267:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.343268:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.343270:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.343271:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.343272:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.343274:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.343275:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.343276:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.343278:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.343279:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.343280:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.343281:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.343282:0:207610:0:(cl_object.c:351:cl_conf_set()) Process entered 00000080:00200000:2.0:1775773861.343283:0:207610:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b15:0x98f:0x0]: losing layout lock 00000080:00000001:2.0:1775773861.343285:0:207610:0:(pcc.c:2213:pcc_layout_invalidate()) Process entered 00000080:00000001:2.0:1775773861.343286:0:207610:0:(pcc.c:2225:pcc_layout_invalidate()) Process leaving 00020000:00000001:2.0:1775773861.343287:0:207610:0:(lov_object.c:1412:lov_conf_set()) Process entered 00020000:00000001:2.0:1775773861.343288:0:207610:0:(lov_object.c:1426:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0) 00020000:00000002:2.0:1775773861.343289:0:207610:0:(lov_object.c:1499:lov_conf_set()) [0x200002b15:0x98f:0x0] lo_layout_invalid=1 00020000:00000001:2.0:1775773861.343291:0:207610:0:(lov_object.c:1500:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343292:0:207610:0:(cl_object.c:359:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.343293:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.343294:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.343295:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.343296:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.343298:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.343299:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.343300:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.343302:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.343303:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.343304:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.343305:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.343307:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.343308:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.343309:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.343311:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.343312:0:207610:0:(file.c:6773:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.343313:0:207610:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:2.0:1775773861.343314:0:207610:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b15:0x98f:0x0](ffff8f621efbe3c8) invalid 00000020:00000001:2.0:1775773861.343315:0:207610:0:(obd_sysfs.c:903:encode_fn_len()) Process filename at ffffaf0701cebca0 00000080:00002000:2.0:1775773861.343316:0:207610:0:(llite_internal.h:2024:d_lustre_invalidate()) invalidate dentry f.103 (ffff8f62191fa948) parent ffff8f6202184798 inode ffff8f621efbe3c8 refc 0 00000080:00000001:2.0:1775773861.343319:0:207610:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:2.0:1775773861.343321:0:207610:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:2.0:1775773861.343323:0:207610:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:2.0:1775773861.343325:0:207610:0:(namei.c:402:ll_lock_cancel_bits()) Process leaving 00000080:00000001:2.0:1775773861.343326:0:207610:0:(namei.c:536:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775773861.343327:0:207610:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1775773861.343328:0:207610:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1775773861.343329:0:207610:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1775773861.343330:0:207610:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1775773861.343331:0:207610:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775773861.343333:0:207610:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8f6226128800 with handle 0xf74d1861c614823 from hash 00010000:00000001:2.0:1775773861.343335:0:207610:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1775773861.343336:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1775773861.343336:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.343337:0:207610:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1775773861.343338:0:207610:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1775773861.343339:0:207610:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:2.0:1775773861.343340:0:207610:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1775773861.343342:0:207610:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62265d7a00/0xf74d1861c61481c lrc: 2/0,0 mode: PW/PW res: [0x200002b15:0x98e:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x800028400000000 nid: local remote: 0xd88660ed39b2cd2d expref: -99 pid: 214091 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00000001:2.0:1775773861.343346:0:207610:0:(namei.c:472:ll_md_blocking_ast()) Process entered 00000080:00000001:2.0:1775773861.343347:0:207610:0:(namei.c:239:ll_lock_cancel_bits()) Process entered 00000080:00000001:2.0:1775773861.343348:0:207610:0:(file.c:6026:ll_have_md_lock()) Process entered 00000080:00000040:2.0:1775773861.343349:0:207610:0:(file.c:6032:ll_have_md_lock()) trying to match res [0x200002b15:0x98e:0x0] mode (null) 00000002:00000001:2.0:1775773861.343351:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.343352:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.343353:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6202d82600 count: 2 00010000:00000001:2.0:1775773861.343354:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.343356:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343357:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.343358:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.343359:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6202d82600 count: 2 00010000:00000001:2.0:1775773861.343360:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.343362:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343363:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.343363:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.343364:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6202d82600 count: 2 00010000:00000001:2.0:1775773861.343365:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.343366:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343367:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.343368:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.343369:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6202d82600 count: 2 00010000:00000001:2.0:1775773861.343370:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.343372:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343373:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.343374:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.343375:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6202d82600 count: 2 00010000:00000001:2.0:1775773861.343376:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.343377:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.343378:0:207610:0:(file.c:6054:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.343379:0:207610:0:(namei.c:211:ll_dom_lock_cancel()) Process entered 00000020:00000001:2.0:1775773861.343380:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.343381:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.343382:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.343384:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.343385:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.343387:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.343388:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.343389:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.343391:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.343392:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.343394:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.343395:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.343397:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.343398:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.343399:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.343401:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.343402:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.343403:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00020000:00000002:2.0:1775773861.343404:0:207610:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff8f621029faa8) owner 0000000000000000/ffff8f62012d8000 00020000:00000001:2.0:1775773861.343406:0:207610:0:(lov_object.c:1110:lov_flush_composite()) Process entered 00000020:00000001:2.0:1775773861.343407:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00000002:00000001:2.0:1775773861.343408:0:207610:0:(mdc_dev.c:306:mdc_dlm_canceling()) Process entered 00000002:00000001:2.0:1775773861.343409:0:207610:0:(mdc_dev.c:255:mdc_lock_flush()) Process entered 00000008:00000001:2.0:1775773861.343409:0:207610:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775773861.343410:0:207610:0:(osc_cache.c:3200:osc_cache_wait_range()) Process entered 00000008:00000020:2.0:1775773861.343412:0:207610:0:(osc_cache.c:3237:osc_cache_wait_range()) obj ffff8f621e5ca540 ready 0|-|- wr 0|-|- rd 0|- sync file range. 00000008:00000001:2.0:1775773861.343414:0:207610:0:(osc_cache.c:3238:osc_cache_wait_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775773861.343415:0:207610:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff8f621e5ca540 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:2.0:1775773861.343418:0:207610:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000002:00000020:2.0:1775773861.343419:0:207610:0:(mdc_dev.c:262:mdc_lock_flush()) object ffff8f621e5ca540: [0 -> 18446744073709551615] 0 pages were written back. 00000002:00000001:2.0:1775773861.343420:0:207610:0:(mdc_dev.c:228:mdc_lock_discard_pages()) Process entered 00000020:00000001:2.0:1775773861.343421:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621e5ca540 -> ffff8f622d9aec38 00000020:00000001:2.0:1775773861.343422:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f622d9aec38 -> ffff8f622d9aec38 00000020:00000001:2.0:1775773861.343423:0:207610:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775773861.343423:0:207610:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775773861.343424:0:207610:0:(vvp_io.c:1817:vvp_io_init()) [0x200002b15:0x98e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775773861.343426:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.343426:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775773861.343427:0:207610:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.343428:0:207610:0:(lov_object.c:1554:lov_io_init()) [0x200002b15:0x98e:0x0]io ffff8f6205260bc8 type 8 ignore/verify layout 1/0 00020000:00000001:2.0:1775773861.343430:0:207610:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775773861.343431:0:207610:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775773861.343432:0:207610:0:(lov_io.c:599:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775773861.343433:0:207610:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775773861.343434:0:207610:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343435:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.343435:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775773861.343436:0:207610:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000001:2.0:1775773861.343436:0:207610:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775773861.343437:0:207610:0:(osc_cache.c:3447:osc_page_gang_lookup()) Process entered 00000008:00000001:2.0:1775773861.343438:0:207610:0:(osc_cache.c:3524:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775773861.343439:0:207610:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775773861.343440:0:207610:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775773861.343441:0:207610:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775773861.343441:0:207610:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775773861.343443:0:207610:0:(vvp_io.c:288:vvp_io_fini()) [0x200002b15:0x98e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775773861.343445:0:207610:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775773861.343445:0:207610:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000002:00000001:2.0:1775773861.343446:0:207610:0:(mdc_dev.c:245:mdc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343447:0:207610:0:(mdc_dev.c:273:mdc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343448:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621e5ca540 -> ffff8f622d9aec38 00000020:00000001:2.0:1775773861.343449:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621e5ca540 -> ffff8f622d9aec38 00000020:00000001:2.0:1775773861.343450:0:207610:0:(cl_object.c:256:cl_object_attr_update()) Process entered 00000008:00000020:2.0:1775773861.343451:0:207610:0:(osc_object.c:161:osc_attr_update()) set kms from 131072to 0 00020000:00000001:2.0:1775773861.343453:0:207610:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:2.0:1775773861.343453:0:207610:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343455:0:207610:0:(cl_object.c:269:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343456:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621e5ca540 -> ffff8f622d9aec38 00000002:00000001:2.0:1775773861.343457:0:207610:0:(mdc_dev.c:346:mdc_dlm_canceling()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343458:0:207610:0:(mdc_dev.c:1535:mdc_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343459:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775773861.343460:0:207610:0:(lov_object.c:1119:lov_flush_composite()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.343462:0:207610:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff8f621029faa8) owner 0000000000000000/ffff8f62012d8000 00000020:00000001:2.0:1775773861.343463:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.343464:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.343465:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.343466:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.343468:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.343480:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.343482:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.343483:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.343485:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.343486:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.343488:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.343490:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.343491:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.343493:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.343494:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.343495:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.343497:0:207610:0:(namei.c:229:ll_dom_lock_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.343498:0:207610:0:(file.c:6731:ll_layout_conf()) Process entered 00000020:00000001:2.0:1775773861.343499:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.343500:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.343502:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.343503:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.343505:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.343506:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.343508:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.343509:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.343511:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.343512:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.343514:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.343515:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.343517:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.343518:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.343519:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.343521:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.343522:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.343523:0:207610:0:(cl_object.c:351:cl_conf_set()) Process entered 00000080:00200000:2.0:1775773861.343523:0:207610:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b15:0x98e:0x0]: losing layout lock 00000080:00000001:2.0:1775773861.343524:0:207610:0:(pcc.c:2213:pcc_layout_invalidate()) Process entered 00000080:00000001:2.0:1775773861.343525:0:207610:0:(pcc.c:2225:pcc_layout_invalidate()) Process leaving 00020000:00000001:2.0:1775773861.343526:0:207610:0:(lov_object.c:1412:lov_conf_set()) Process entered 00020000:00000001:2.0:1775773861.343527:0:207610:0:(lov_object.c:1426:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0) 00020000:00000002:2.0:1775773861.343528:0:207610:0:(lov_object.c:1499:lov_conf_set()) [0x200002b15:0x98e:0x0] lo_layout_invalid=1 00020000:00000001:2.0:1775773861.343529:0:207610:0:(lov_object.c:1500:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343530:0:207610:0:(cl_object.c:359:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.343531:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.343532:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.343534:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.343535:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.343537:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.343538:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.343539:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.343541:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.343542:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.343543:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.343545:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.343546:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.343547:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.343548:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.343550:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.343551:0:207610:0:(file.c:6773:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.343552:0:207610:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:2.0:1775773861.343553:0:207610:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b15:0x98e:0x0](ffff8f621efb5348) invalid 00000020:00000001:2.0:1775773861.343555:0:207610:0:(obd_sysfs.c:903:encode_fn_len()) Process filename at ffffaf0701cebca0 00000080:00002000:2.0:1775773861.343556:0:207610:0:(llite_internal.h:2024:d_lustre_invalidate()) invalidate dentry f.102 (ffff8f62094a0288) parent ffff8f6202184798 inode ffff8f621efb5348 refc 0 00000080:00000001:2.0:1775773861.343559:0:207610:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:2.0:1775773861.343561:0:207610:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:2.0:1775773861.343563:0:207610:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:2.0:1775773861.343564:0:207610:0:(namei.c:402:ll_lock_cancel_bits()) Process leaving 00000080:00000001:2.0:1775773861.343565:0:207610:0:(namei.c:536:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775773861.343567:0:207610:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1775773861.343568:0:207610:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1775773861.343569:0:207610:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1775773861.343570:0:207610:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1775773861.343571:0:207610:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775773861.343573:0:207610:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8f62265d7a00 with handle 0xf74d1861c61481c from hash 00010000:00000001:2.0:1775773861.343574:0:207610:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1775773861.343576:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1775773861.343576:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.343577:0:207610:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1775773861.343578:0:207610:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1775773861.343579:0:207610:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:2.0:1775773861.343581:0:207610:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1775773861.343583:0:207610:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226129800/0xf74d1861c614815 lrc: 2/0,0 mode: PW/PW res: [0x200002b15:0x98d:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x800028400000000 nid: local remote: 0xd88660ed39b2cd0a expref: -99 pid: 214090 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00000001:2.0:1775773861.343587:0:207610:0:(namei.c:472:ll_md_blocking_ast()) Process entered 00000080:00000001:2.0:1775773861.343588:0:207610:0:(namei.c:239:ll_lock_cancel_bits()) Process entered 00000080:00000001:2.0:1775773861.343589:0:207610:0:(file.c:6026:ll_have_md_lock()) Process entered 00000080:00000040:2.0:1775773861.343590:0:207610:0:(file.c:6032:ll_have_md_lock()) trying to match res [0x200002b15:0x98d:0x0] mode (null) 00000002:00000001:2.0:1775773861.343591:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.343592:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.343593:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960700 count: 2 00010000:00000001:2.0:1775773861.343594:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.343596:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343597:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.343598:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.343599:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960700 count: 2 00010000:00000001:2.0:1775773861.343600:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.343602:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343603:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.343603:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.343604:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960700 count: 2 00010000:00000001:2.0:1775773861.343605:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.343607:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343608:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.343608:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.343609:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960700 count: 2 00010000:00000001:2.0:1775773861.343610:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.343611:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343612:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.343613:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.343614:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960700 count: 2 00010000:00000001:2.0:1775773861.343615:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.343616:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.343617:0:207610:0:(file.c:6054:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.343618:0:207610:0:(namei.c:211:ll_dom_lock_cancel()) Process entered 00000020:00000001:2.0:1775773861.343619:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.343621:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.343623:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.343626:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.343628:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.343631:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.343633:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.343635:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.343638:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.343640:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.343642:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.343644:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.343647:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.343649:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.343652:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.343654:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.343656:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.343658:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00020000:00000002:2.0:1775773861.343660:0:207610:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff8f622e1969b0) owner 0000000000000000/ffff8f62012d8000 00020000:00000001:2.0:1775773861.343663:0:207610:0:(lov_object.c:1110:lov_flush_composite()) Process entered 00000020:00000001:2.0:1775773861.343664:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00000002:00000001:2.0:1775773861.343666:0:207610:0:(mdc_dev.c:306:mdc_dlm_canceling()) Process entered 00000002:00000001:2.0:1775773861.343668:0:207610:0:(mdc_dev.c:255:mdc_lock_flush()) Process entered 00000008:00000001:2.0:1775773861.343669:0:207610:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775773861.343671:0:207610:0:(osc_cache.c:3200:osc_cache_wait_range()) Process entered 00000008:00000020:2.0:1775773861.343673:0:207610:0:(osc_cache.c:3237:osc_cache_wait_range()) obj ffff8f623476f180 ready 0|-|- wr 0|-|- rd 0|- sync file range. 00000008:00000001:2.0:1775773861.343675:0:207610:0:(osc_cache.c:3238:osc_cache_wait_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775773861.343677:0:207610:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff8f623476f180 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:2.0:1775773861.343680:0:207610:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000002:00000020:2.0:1775773861.343682:0:207610:0:(mdc_dev.c:262:mdc_lock_flush()) object ffff8f623476f180: [0 -> 18446744073709551615] 0 pages were written back. 00000002:00000001:2.0:1775773861.343683:0:207610:0:(mdc_dev.c:228:mdc_lock_discard_pages()) Process entered 00000020:00000001:2.0:1775773861.343685:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476f180 -> ffff8f622c2e03a0 00000020:00000001:2.0:1775773861.343686:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f622c2e03a0 -> ffff8f622c2e03a0 00000020:00000001:2.0:1775773861.343688:0:207610:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775773861.343689:0:207610:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775773861.343690:0:207610:0:(vvp_io.c:1817:vvp_io_init()) [0x200002b15:0x98d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775773861.343692:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.343693:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775773861.343694:0:207610:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.343697:0:207610:0:(lov_object.c:1554:lov_io_init()) [0x200002b15:0x98d:0x0]io ffff8f6205260bc8 type 8 ignore/verify layout 1/0 00020000:00000001:2.0:1775773861.343699:0:207610:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775773861.343700:0:207610:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775773861.343702:0:207610:0:(lov_io.c:599:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775773861.343703:0:207610:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775773861.343704:0:207610:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343706:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.343707:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775773861.343708:0:207610:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000001:2.0:1775773861.343709:0:207610:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775773861.343710:0:207610:0:(osc_cache.c:3447:osc_page_gang_lookup()) Process entered 00000008:00000001:2.0:1775773861.343712:0:207610:0:(osc_cache.c:3524:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775773861.343714:0:207610:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775773861.343715:0:207610:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775773861.343716:0:207610:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775773861.343717:0:207610:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775773861.343719:0:207610:0:(vvp_io.c:288:vvp_io_fini()) [0x200002b15:0x98d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775773861.343722:0:207610:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775773861.343723:0:207610:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000002:00000001:2.0:1775773861.343724:0:207610:0:(mdc_dev.c:245:mdc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343725:0:207610:0:(mdc_dev.c:273:mdc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343727:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476f180 -> ffff8f622c2e03a0 00000020:00000001:2.0:1775773861.343728:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476f180 -> ffff8f622c2e03a0 00000020:00000001:2.0:1775773861.343730:0:207610:0:(cl_object.c:256:cl_object_attr_update()) Process entered 00000008:00000020:2.0:1775773861.343732:0:207610:0:(osc_object.c:161:osc_attr_update()) set kms from 131072to 0 00020000:00000001:2.0:1775773861.343734:0:207610:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:2.0:1775773861.343735:0:207610:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343737:0:207610:0:(cl_object.c:269:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343738:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476f180 -> ffff8f622c2e03a0 00000002:00000001:2.0:1775773861.343741:0:207610:0:(mdc_dev.c:346:mdc_dlm_canceling()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343743:0:207610:0:(mdc_dev.c:1535:mdc_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343744:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775773861.343746:0:207610:0:(lov_object.c:1119:lov_flush_composite()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.343748:0:207610:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff8f622e1969b0) owner 0000000000000000/ffff8f62012d8000 00000020:00000001:2.0:1775773861.343750:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.343752:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.343754:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.343756:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.343759:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.343762:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.343765:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.343767:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.343770:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.343772:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.343773:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.343775:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.343776:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.343778:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.343780:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.343782:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.343784:0:207610:0:(namei.c:229:ll_dom_lock_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.343785:0:207610:0:(file.c:6731:ll_layout_conf()) Process entered 00000020:00000001:2.0:1775773861.343786:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.343788:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.343790:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.343792:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.343793:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.343795:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.343797:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.343798:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.343800:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.343801:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.343803:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.343805:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.343806:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.343808:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.343809:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.343811:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.343812:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.343813:0:207610:0:(cl_object.c:351:cl_conf_set()) Process entered 00000080:00200000:2.0:1775773861.343814:0:207610:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b15:0x98d:0x0]: losing layout lock 00000080:00000001:2.0:1775773861.343815:0:207610:0:(pcc.c:2213:pcc_layout_invalidate()) Process entered 00000080:00000001:2.0:1775773861.343816:0:207610:0:(pcc.c:2225:pcc_layout_invalidate()) Process leaving 00020000:00000001:2.0:1775773861.343817:0:207610:0:(lov_object.c:1412:lov_conf_set()) Process entered 00020000:00000001:2.0:1775773861.343819:0:207610:0:(lov_object.c:1426:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0) 00020000:00000002:2.0:1775773861.343820:0:207610:0:(lov_object.c:1499:lov_conf_set()) [0x200002b15:0x98d:0x0] lo_layout_invalid=1 00020000:00000001:2.0:1775773861.343821:0:207610:0:(lov_object.c:1500:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343822:0:207610:0:(cl_object.c:359:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.343823:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.343824:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.343825:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.343827:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.343828:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.343830:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.343831:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.343833:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.343834:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.343836:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.343837:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.343838:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.343839:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.343841:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.343842:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.343843:0:207610:0:(file.c:6773:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.343844:0:207610:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:2.0:1775773861.343845:0:207610:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b15:0x98d:0x0](ffff8f621efbcb08) invalid 00000020:00000001:2.0:1775773861.343847:0:207610:0:(obd_sysfs.c:903:encode_fn_len()) Process filename at ffffaf0701cebca0 00000080:00002000:2.0:1775773861.343848:0:207610:0:(llite_internal.h:2024:d_lustre_invalidate()) invalidate dentry f.101 (ffff8f62191fa870) parent ffff8f6202184798 inode ffff8f621efbcb08 refc 0 00000080:00000001:2.0:1775773861.343851:0:207610:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:2.0:1775773861.343854:0:207610:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:2.0:1775773861.343856:0:207610:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:2.0:1775773861.343857:0:207610:0:(namei.c:402:ll_lock_cancel_bits()) Process leaving 00000080:00000001:2.0:1775773861.343858:0:207610:0:(namei.c:536:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775773861.343860:0:207610:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1775773861.343861:0:207610:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1775773861.343862:0:207610:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1775773861.343863:0:207610:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1775773861.343864:0:207610:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775773861.343866:0:207610:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8f6226129800 with handle 0xf74d1861c614815 from hash 00010000:00000001:2.0:1775773861.343868:0:207610:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1775773861.343869:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1775773861.343870:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.343870:0:207610:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1775773861.343872:0:207610:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1775773861.343873:0:207610:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:2.0:1775773861.343875:0:207610:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1775773861.343877:0:207610:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476d000/0xf74d1861c61480e lrc: 2/0,0 mode: PW/PW res: [0x200002b15:0x98c:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x800028400000000 nid: local remote: 0xd88660ed39b2cce7 expref: -99 pid: 214089 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00000001:2.0:1775773861.343882:0:207610:0:(namei.c:472:ll_md_blocking_ast()) Process entered 00000080:00000001:2.0:1775773861.343883:0:207610:0:(namei.c:239:ll_lock_cancel_bits()) Process entered 00000080:00000001:2.0:1775773861.343884:0:207610:0:(file.c:6026:ll_have_md_lock()) Process entered 00000080:00000040:2.0:1775773861.343885:0:207610:0:(file.c:6032:ll_have_md_lock()) trying to match res [0x200002b15:0x98c:0x0] mode (null) 00000002:00000001:2.0:1775773861.343886:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.343887:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.343889:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6206bb7a00 count: 2 00010000:00000001:2.0:1775773861.343890:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.343892:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343893:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.343894:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.343895:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6206bb7a00 count: 2 00010000:00000001:2.0:1775773861.343896:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.343897:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343898:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.343899:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.343900:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6206bb7a00 count: 2 00010000:00000001:2.0:1775773861.343901:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.343902:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343903:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.343904:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.343904:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6206bb7a00 count: 2 00010000:00000001:2.0:1775773861.343905:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.343907:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343907:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.343908:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.343909:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6206bb7a00 count: 2 00010000:00000001:2.0:1775773861.343910:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.343911:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.343912:0:207610:0:(file.c:6054:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.343913:0:207610:0:(namei.c:211:ll_dom_lock_cancel()) Process entered 00000020:00000001:2.0:1775773861.343914:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.343915:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.343916:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.343918:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.343920:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.343922:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.343923:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.343924:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.343926:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.343927:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.343929:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.343930:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.343932:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.343933:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.343935:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.343936:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.343937:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.343938:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00020000:00000002:2.0:1775773861.343939:0:207610:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff8f622c0412e8) owner 0000000000000000/ffff8f62012d8000 00020000:00000001:2.0:1775773861.343941:0:207610:0:(lov_object.c:1110:lov_flush_composite()) Process entered 00000020:00000001:2.0:1775773861.343942:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00000002:00000001:2.0:1775773861.343943:0:207610:0:(mdc_dev.c:306:mdc_dlm_canceling()) Process entered 00000002:00000001:2.0:1775773861.343944:0:207610:0:(mdc_dev.c:255:mdc_lock_flush()) Process entered 00000008:00000001:2.0:1775773861.343945:0:207610:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775773861.343946:0:207610:0:(osc_cache.c:3200:osc_cache_wait_range()) Process entered 00000008:00000020:2.0:1775773861.343947:0:207610:0:(osc_cache.c:3237:osc_cache_wait_range()) obj ffff8f621ddb5dc0 ready 0|-|- wr 0|-|- rd 0|- sync file range. 00000008:00000001:2.0:1775773861.343949:0:207610:0:(osc_cache.c:3238:osc_cache_wait_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775773861.343951:0:207610:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff8f621ddb5dc0 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:2.0:1775773861.343953:0:207610:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000002:00000020:2.0:1775773861.343954:0:207610:0:(mdc_dev.c:262:mdc_lock_flush()) object ffff8f621ddb5dc0: [0 -> 18446744073709551615] 0 pages were written back. 00000002:00000001:2.0:1775773861.343956:0:207610:0:(mdc_dev.c:228:mdc_lock_discard_pages()) Process entered 00000020:00000001:2.0:1775773861.343957:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621ddb5dc0 -> ffff8f6233999dc8 00000020:00000001:2.0:1775773861.343958:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f6233999dc8 -> ffff8f6233999dc8 00000020:00000001:2.0:1775773861.343959:0:207610:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775773861.343960:0:207610:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775773861.343961:0:207610:0:(vvp_io.c:1817:vvp_io_init()) [0x200002b15:0x98c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775773861.343962:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.343963:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775773861.343964:0:207610:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.343966:0:207610:0:(lov_object.c:1554:lov_io_init()) [0x200002b15:0x98c:0x0]io ffff8f6205260bc8 type 8 ignore/verify layout 1/0 00020000:00000001:2.0:1775773861.343968:0:207610:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775773861.343969:0:207610:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775773861.343970:0:207610:0:(lov_io.c:599:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775773861.343971:0:207610:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775773861.343971:0:207610:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343972:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.343973:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775773861.343974:0:207610:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000001:2.0:1775773861.343974:0:207610:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775773861.343975:0:207610:0:(osc_cache.c:3447:osc_page_gang_lookup()) Process entered 00000008:00000001:2.0:1775773861.343977:0:207610:0:(osc_cache.c:3524:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775773861.343978:0:207610:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775773861.343978:0:207610:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775773861.343979:0:207610:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775773861.343980:0:207610:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775773861.343981:0:207610:0:(vvp_io.c:288:vvp_io_fini()) [0x200002b15:0x98c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775773861.343982:0:207610:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775773861.343983:0:207610:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000002:00000001:2.0:1775773861.343984:0:207610:0:(mdc_dev.c:245:mdc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343985:0:207610:0:(mdc_dev.c:273:mdc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343986:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621ddb5dc0 -> ffff8f6233999dc8 00000020:00000001:2.0:1775773861.343987:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621ddb5dc0 -> ffff8f6233999dc8 00000020:00000001:2.0:1775773861.343988:0:207610:0:(cl_object.c:256:cl_object_attr_update()) Process entered 00000008:00000020:2.0:1775773861.343989:0:207610:0:(osc_object.c:161:osc_attr_update()) set kms from 131072to 0 00020000:00000001:2.0:1775773861.343990:0:207610:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:2.0:1775773861.343991:0:207610:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343992:0:207610:0:(cl_object.c:269:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343993:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621ddb5dc0 -> ffff8f6233999dc8 00000002:00000001:2.0:1775773861.343995:0:207610:0:(mdc_dev.c:346:mdc_dlm_canceling()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.343996:0:207610:0:(mdc_dev.c:1535:mdc_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.343997:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775773861.343998:0:207610:0:(lov_object.c:1119:lov_flush_composite()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.343999:0:207610:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff8f622c0412e8) owner 0000000000000000/ffff8f62012d8000 00000020:00000001:2.0:1775773861.344000:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.344002:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.344003:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.344005:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.344007:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.344008:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.344010:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.344011:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.344013:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.344014:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.344016:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.344017:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.344018:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.344019:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.344021:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.344022:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.344023:0:207610:0:(namei.c:229:ll_dom_lock_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.344025:0:207610:0:(file.c:6731:ll_layout_conf()) Process entered 00000020:00000001:2.0:1775773861.344025:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.344027:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.344029:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.344030:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.344031:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.344033:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.344034:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.344036:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.344037:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.344039:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.344040:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.344042:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.344043:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.344044:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.344045:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.344047:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.344048:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.344049:0:207610:0:(cl_object.c:351:cl_conf_set()) Process entered 00000080:00200000:2.0:1775773861.344050:0:207610:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b15:0x98c:0x0]: losing layout lock 00000080:00000001:2.0:1775773861.344051:0:207610:0:(pcc.c:2213:pcc_layout_invalidate()) Process entered 00000080:00000001:2.0:1775773861.344052:0:207610:0:(pcc.c:2225:pcc_layout_invalidate()) Process leaving 00020000:00000001:2.0:1775773861.344053:0:207610:0:(lov_object.c:1412:lov_conf_set()) Process entered 00020000:00000001:2.0:1775773861.344054:0:207610:0:(lov_object.c:1426:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0) 00020000:00000002:2.0:1775773861.344055:0:207610:0:(lov_object.c:1499:lov_conf_set()) [0x200002b15:0x98c:0x0] lo_layout_invalid=1 00020000:00000001:2.0:1775773861.344056:0:207610:0:(lov_object.c:1500:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.344058:0:207610:0:(cl_object.c:359:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.344059:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.344060:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.344061:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.344062:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.344064:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.344065:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.344066:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.344067:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.344069:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.344070:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.344071:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.344072:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.344074:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.344075:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.344076:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.344078:0:207610:0:(file.c:6773:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.344079:0:207610:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:2.0:1775773861.344080:0:207610:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b15:0x98c:0x0](ffff8f6226a180c8) invalid 00000020:00000001:2.0:1775773861.344081:0:207610:0:(obd_sysfs.c:903:encode_fn_len()) Process filename at ffffaf0701cebca0 00000080:00002000:2.0:1775773861.344082:0:207610:0:(llite_internal.h:2024:d_lustre_invalidate()) invalidate dentry f.100 (ffff8f6209b2da20) parent ffff8f6202184798 inode ffff8f6226a180c8 refc 0 00000080:00000001:2.0:1775773861.344085:0:207610:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:2.0:1775773861.344087:0:207610:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:2.0:1775773861.344089:0:207610:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:2.0:1775773861.344090:0:207610:0:(namei.c:402:ll_lock_cancel_bits()) Process leaving 00000080:00000001:2.0:1775773861.344091:0:207610:0:(namei.c:536:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775773861.344092:0:207610:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1775773861.344093:0:207610:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1775773861.344094:0:207610:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1775773861.344095:0:207610:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1775773861.344097:0:207610:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775773861.344099:0:207610:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8f623476d000 with handle 0xf74d1861c61480e from hash 00010000:00000001:2.0:1775773861.344100:0:207610:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1775773861.344101:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1775773861.344102:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.344102:0:207610:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1775773861.344103:0:207610:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1775773861.344104:0:207610:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:2.0:1775773861.344106:0:207610:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1775773861.344107:0:207610:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226128600/0xf74d1861c614807 lrc: 2/0,0 mode: PW/PW res: [0x200002b15:0x98b:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x800028400000000 nid: local remote: 0xd88660ed39b2ccc4 expref: -99 pid: 214088 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00000001:2.0:1775773861.344111:0:207610:0:(namei.c:472:ll_md_blocking_ast()) Process entered 00000080:00000001:2.0:1775773861.344113:0:207610:0:(namei.c:239:ll_lock_cancel_bits()) Process entered 00000080:00000001:2.0:1775773861.344113:0:207610:0:(file.c:6026:ll_have_md_lock()) Process entered 00000080:00000040:2.0:1775773861.344114:0:207610:0:(file.c:6032:ll_have_md_lock()) trying to match res [0x200002b15:0x98b:0x0] mode (null) 00000002:00000001:2.0:1775773861.344116:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.344116:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.344118:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f622d964200 count: 2 00010000:00000001:2.0:1775773861.344119:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.344120:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.344122:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.344123:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.344123:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f622d964200 count: 2 00010000:00000001:2.0:1775773861.344125:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.344126:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.344127:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.344128:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.344129:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f622d964200 count: 2 00010000:00000001:2.0:1775773861.344130:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.344131:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.344132:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.344147:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.344148:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f622d964200 count: 2 00010000:00000001:2.0:1775773861.344150:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.344152:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.344153:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.344155:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.344156:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f622d964200 count: 2 00010000:00000001:2.0:1775773861.344158:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.344159:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.344160:0:207610:0:(file.c:6054:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.344161:0:207610:0:(namei.c:211:ll_dom_lock_cancel()) Process entered 00000020:00000001:2.0:1775773861.344162:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.344164:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.344166:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.344168:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.344170:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.344172:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.344173:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.344175:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.344177:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.344179:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.344181:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.344183:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.344185:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.344187:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.344188:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.344190:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.344192:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.344193:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00020000:00000002:2.0:1775773861.344195:0:207610:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff8f621029fd90) owner 0000000000000000/ffff8f62012d8000 00020000:00000001:2.0:1775773861.344196:0:207610:0:(lov_object.c:1110:lov_flush_composite()) Process entered 00000020:00000001:2.0:1775773861.344198:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00000002:00000001:2.0:1775773861.344199:0:207610:0:(mdc_dev.c:306:mdc_dlm_canceling()) Process entered 00000002:00000001:2.0:1775773861.344200:0:207610:0:(mdc_dev.c:255:mdc_lock_flush()) Process entered 00000008:00000001:2.0:1775773861.344201:0:207610:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775773861.344203:0:207610:0:(osc_cache.c:3200:osc_cache_wait_range()) Process entered 00000008:00000020:2.0:1775773861.344204:0:207610:0:(osc_cache.c:3237:osc_cache_wait_range()) obj ffff8f621e5cba40 ready 0|-|- wr 0|-|- rd 0|- sync file range. 00000008:00000001:2.0:1775773861.344206:0:207610:0:(osc_cache.c:3238:osc_cache_wait_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775773861.344208:0:207610:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff8f621e5cba40 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:2.0:1775773861.344210:0:207610:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000002:00000020:2.0:1775773861.344211:0:207610:0:(mdc_dev.c:262:mdc_lock_flush()) object ffff8f621e5cba40: [0 -> 18446744073709551615] 0 pages were written back. 00000002:00000001:2.0:1775773861.344213:0:207610:0:(mdc_dev.c:228:mdc_lock_discard_pages()) Process entered 00000020:00000001:2.0:1775773861.344214:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621e5cba40 -> ffff8f622d9ae080 00000020:00000001:2.0:1775773861.344214:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f622d9ae080 -> ffff8f622d9ae080 00000020:00000001:2.0:1775773861.344215:0:207610:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775773861.344216:0:207610:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775773861.344217:0:207610:0:(vvp_io.c:1817:vvp_io_init()) [0x200002b15:0x98b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775773861.344218:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.344219:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775773861.344220:0:207610:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.344221:0:207610:0:(lov_object.c:1554:lov_io_init()) [0x200002b15:0x98b:0x0]io ffff8f6205260bc8 type 8 ignore/verify layout 1/0 00020000:00000001:2.0:1775773861.344223:0:207610:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775773861.344223:0:207610:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775773861.344224:0:207610:0:(lov_io.c:599:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775773861.344225:0:207610:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775773861.344226:0:207610:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.344227:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.344228:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775773861.344229:0:207610:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000001:2.0:1775773861.344230:0:207610:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775773861.344231:0:207610:0:(osc_cache.c:3447:osc_page_gang_lookup()) Process entered 00000008:00000001:2.0:1775773861.344232:0:207610:0:(osc_cache.c:3524:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775773861.344233:0:207610:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775773861.344234:0:207610:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775773861.344234:0:207610:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775773861.344235:0:207610:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775773861.344236:0:207610:0:(vvp_io.c:288:vvp_io_fini()) [0x200002b15:0x98b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775773861.344238:0:207610:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775773861.344238:0:207610:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000002:00000001:2.0:1775773861.344239:0:207610:0:(mdc_dev.c:245:mdc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.344240:0:207610:0:(mdc_dev.c:273:mdc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.344241:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621e5cba40 -> ffff8f622d9ae080 00000020:00000001:2.0:1775773861.344242:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621e5cba40 -> ffff8f622d9ae080 00000020:00000001:2.0:1775773861.344243:0:207610:0:(cl_object.c:256:cl_object_attr_update()) Process entered 00000008:00000020:2.0:1775773861.344244:0:207610:0:(osc_object.c:161:osc_attr_update()) set kms from 131072to 0 00020000:00000001:2.0:1775773861.344245:0:207610:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:2.0:1775773861.344246:0:207610:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.344247:0:207610:0:(cl_object.c:269:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.344248:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621e5cba40 -> ffff8f622d9ae080 00000002:00000001:2.0:1775773861.344250:0:207610:0:(mdc_dev.c:346:mdc_dlm_canceling()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.344251:0:207610:0:(mdc_dev.c:1535:mdc_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.344252:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775773861.344253:0:207610:0:(lov_object.c:1119:lov_flush_composite()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.344255:0:207610:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff8f621029fd90) owner 0000000000000000/ffff8f62012d8000 00000020:00000001:2.0:1775773861.344256:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.344257:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.344259:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.344260:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.344262:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.344263:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.344265:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.344266:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.344268:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.344270:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.344272:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.344273:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.344274:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.344276:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.344278:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.344279:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.344280:0:207610:0:(namei.c:229:ll_dom_lock_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.344281:0:207610:0:(file.c:6731:ll_layout_conf()) Process entered 00000020:00000001:2.0:1775773861.344282:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.344283:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.344285:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.344286:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.344287:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.344289:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.344290:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.344292:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.344293:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.344295:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.344296:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.344298:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.344299:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.344300:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.344302:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.344303:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.344304:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.344305:0:207610:0:(cl_object.c:351:cl_conf_set()) Process entered 00000080:00200000:2.0:1775773861.344306:0:207610:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b15:0x98b:0x0]: losing layout lock 00000080:00000001:2.0:1775773861.344307:0:207610:0:(pcc.c:2213:pcc_layout_invalidate()) Process entered 00000080:00000001:2.0:1775773861.344308:0:207610:0:(pcc.c:2225:pcc_layout_invalidate()) Process leaving 00020000:00000001:2.0:1775773861.344308:0:207610:0:(lov_object.c:1412:lov_conf_set()) Process entered 00020000:00000001:2.0:1775773861.344309:0:207610:0:(lov_object.c:1426:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0) 00020000:00000002:2.0:1775773861.344310:0:207610:0:(lov_object.c:1499:lov_conf_set()) [0x200002b15:0x98b:0x0] lo_layout_invalid=1 00020000:00000001:2.0:1775773861.344312:0:207610:0:(lov_object.c:1500:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.344313:0:207610:0:(cl_object.c:359:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.344313:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.344314:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.344316:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.344317:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.344318:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.344319:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.344321:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.344322:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.344324:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.344325:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.344327:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.344328:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.344330:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.344331:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.344332:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.344334:0:207610:0:(file.c:6773:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.344335:0:207610:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:2.0:1775773861.344336:0:207610:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b15:0x98b:0x0](ffff8f621efb3a88) invalid 00000020:00000001:2.0:1775773861.344338:0:207610:0:(obd_sysfs.c:903:encode_fn_len()) Process filename at ffffaf0701cebca0 00000080:00002000:2.0:1775773861.344339:0:207610:0:(llite_internal.h:2024:d_lustre_invalidate()) invalidate dentry f.99 (ffff8f62191fa360) parent ffff8f6202184798 inode ffff8f621efb3a88 refc 0 00000080:00000001:2.0:1775773861.344342:0:207610:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:2.0:1775773861.344344:0:207610:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:2.0:1775773861.344346:0:207610:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:2.0:1775773861.344347:0:207610:0:(namei.c:402:ll_lock_cancel_bits()) Process leaving 00000080:00000001:2.0:1775773861.344348:0:207610:0:(namei.c:536:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775773861.344350:0:207610:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1775773861.344351:0:207610:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1775773861.344352:0:207610:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1775773861.344353:0:207610:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1775773861.344354:0:207610:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775773861.344356:0:207610:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8f6226128600 with handle 0xf74d1861c614807 from hash 00010000:00000001:2.0:1775773861.344357:0:207610:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1775773861.344358:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1775773861.344359:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.344360:0:207610:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1775773861.344361:0:207610:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1775773861.344361:0:207610:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:2.0:1775773861.344363:0:207610:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1775773861.344365:0:207610:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f621839e600/0xf74d1861c614800 lrc: 2/0,0 mode: PW/PW res: [0x200002b15:0x98a:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x800028400000000 nid: local remote: 0xd88660ed39b2cca1 expref: -99 pid: 214087 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00000001:2.0:1775773861.344369:0:207610:0:(namei.c:472:ll_md_blocking_ast()) Process entered 00000080:00000001:2.0:1775773861.344371:0:207610:0:(namei.c:239:ll_lock_cancel_bits()) Process entered 00000080:00000001:2.0:1775773861.344372:0:207610:0:(file.c:6026:ll_have_md_lock()) Process entered 00000080:00000040:2.0:1775773861.344373:0:207610:0:(file.c:6032:ll_have_md_lock()) trying to match res [0x200002b15:0x98a:0x0] mode (null) 00000002:00000001:2.0:1775773861.344376:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.344377:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.344379:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f622d964d00 count: 2 00010000:00000001:2.0:1775773861.344381:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.344384:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.344386:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.344387:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.344388:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f622d964d00 count: 2 00010000:00000001:2.0:1775773861.344390:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.344393:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.344395:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.344396:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.344397:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f622d964d00 count: 2 00010000:00000001:2.0:1775773861.344399:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.344401:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.344403:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.344404:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.344406:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f622d964d00 count: 2 00010000:00000001:2.0:1775773861.344408:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.344410:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.344411:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.344412:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.344414:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f622d964d00 count: 2 00010000:00000001:2.0:1775773861.344416:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.344418:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.344419:0:207610:0:(file.c:6054:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.344421:0:207610:0:(namei.c:211:ll_dom_lock_cancel()) Process entered 00000020:00000001:2.0:1775773861.344422:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.344425:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.344428:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.344430:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.344433:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.344435:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.344438:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.344440:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.344442:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.344444:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.344445:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.344447:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.344448:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.344450:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.344452:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.344454:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.344455:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.344456:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00020000:00000002:2.0:1775773861.344458:0:207610:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff8f621029f0f8) owner 0000000000000000/ffff8f62012d8000 00020000:00000001:2.0:1775773861.344459:0:207610:0:(lov_object.c:1110:lov_flush_composite()) Process entered 00000020:00000001:2.0:1775773861.344460:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00000002:00000001:2.0:1775773861.344462:0:207610:0:(mdc_dev.c:306:mdc_dlm_canceling()) Process entered 00000002:00000001:2.0:1775773861.344463:0:207610:0:(mdc_dev.c:255:mdc_lock_flush()) Process entered 00000008:00000001:2.0:1775773861.344464:0:207610:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775773861.344465:0:207610:0:(osc_cache.c:3200:osc_cache_wait_range()) Process entered 00000008:00000020:2.0:1775773861.344467:0:207610:0:(osc_cache.c:3237:osc_cache_wait_range()) obj ffff8f621e5ca1c0 ready 0|-|- wr 0|-|- rd 0|- sync file range. 00000008:00000001:2.0:1775773861.344468:0:207610:0:(osc_cache.c:3238:osc_cache_wait_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775773861.344483:0:207610:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff8f621e5ca1c0 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:2.0:1775773861.344486:0:207610:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000002:00000020:2.0:1775773861.344487:0:207610:0:(mdc_dev.c:262:mdc_lock_flush()) object ffff8f621e5ca1c0: [0 -> 18446744073709551615] 0 pages were written back. 00000002:00000001:2.0:1775773861.344489:0:207610:0:(mdc_dev.c:228:mdc_lock_discard_pages()) Process entered 00000020:00000001:2.0:1775773861.344490:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621e5ca1c0 -> ffff8f622d9aed00 00000020:00000001:2.0:1775773861.344491:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f622d9aed00 -> ffff8f622d9aed00 00000020:00000001:2.0:1775773861.344492:0:207610:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775773861.344493:0:207610:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775773861.344494:0:207610:0:(vvp_io.c:1817:vvp_io_init()) [0x200002b15:0x98a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775773861.344495:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.344496:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775773861.344497:0:207610:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.344499:0:207610:0:(lov_object.c:1554:lov_io_init()) [0x200002b15:0x98a:0x0]io ffff8f6205260bc8 type 8 ignore/verify layout 1/0 00020000:00000001:2.0:1775773861.344500:0:207610:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775773861.344501:0:207610:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775773861.344502:0:207610:0:(lov_io.c:599:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775773861.344503:0:207610:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775773861.344504:0:207610:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.344505:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.344506:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775773861.344506:0:207610:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000001:2.0:1775773861.344507:0:207610:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775773861.344508:0:207610:0:(osc_cache.c:3447:osc_page_gang_lookup()) Process entered 00000008:00000001:2.0:1775773861.344510:0:207610:0:(osc_cache.c:3524:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775773861.344511:0:207610:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775773861.344512:0:207610:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775773861.344513:0:207610:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775773861.344514:0:207610:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775773861.344515:0:207610:0:(vvp_io.c:288:vvp_io_fini()) [0x200002b15:0x98a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775773861.344517:0:207610:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775773861.344517:0:207610:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000002:00000001:2.0:1775773861.344518:0:207610:0:(mdc_dev.c:245:mdc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.344519:0:207610:0:(mdc_dev.c:273:mdc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.344520:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621e5ca1c0 -> ffff8f622d9aed00 00000020:00000001:2.0:1775773861.344521:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621e5ca1c0 -> ffff8f622d9aed00 00000020:00000001:2.0:1775773861.344522:0:207610:0:(cl_object.c:256:cl_object_attr_update()) Process entered 00000008:00000020:2.0:1775773861.344523:0:207610:0:(osc_object.c:161:osc_attr_update()) set kms from 131072to 0 00020000:00000001:2.0:1775773861.344524:0:207610:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:2.0:1775773861.344525:0:207610:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.344526:0:207610:0:(cl_object.c:269:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.344527:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f621e5ca1c0 -> ffff8f622d9aed00 00000002:00000001:2.0:1775773861.344529:0:207610:0:(mdc_dev.c:346:mdc_dlm_canceling()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.344530:0:207610:0:(mdc_dev.c:1535:mdc_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.344531:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775773861.344532:0:207610:0:(lov_object.c:1119:lov_flush_composite()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.344533:0:207610:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff8f621029f0f8) owner 0000000000000000/ffff8f62012d8000 00000020:00000001:2.0:1775773861.344535:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.344536:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.344538:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.344540:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.344541:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.344543:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.344544:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.344546:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.344548:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.344549:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.344551:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.344552:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.344554:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.344556:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.344557:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.344559:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.344561:0:207610:0:(namei.c:229:ll_dom_lock_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.344562:0:207610:0:(file.c:6731:ll_layout_conf()) Process entered 00000020:00000001:2.0:1775773861.344563:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.344564:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.344566:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.344568:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.344569:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.344571:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.344572:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.344574:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.344575:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.344577:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.344578:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.344580:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.344581:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.344582:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.344583:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.344585:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.344586:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.344587:0:207610:0:(cl_object.c:351:cl_conf_set()) Process entered 00000080:00200000:2.0:1775773861.344587:0:207610:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b15:0x98a:0x0]: losing layout lock 00000080:00000001:2.0:1775773861.344589:0:207610:0:(pcc.c:2213:pcc_layout_invalidate()) Process entered 00000080:00000001:2.0:1775773861.344590:0:207610:0:(pcc.c:2225:pcc_layout_invalidate()) Process leaving 00020000:00000001:2.0:1775773861.344591:0:207610:0:(lov_object.c:1412:lov_conf_set()) Process entered 00020000:00000001:2.0:1775773861.344592:0:207610:0:(lov_object.c:1426:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0) 00020000:00000002:2.0:1775773861.344593:0:207610:0:(lov_object.c:1499:lov_conf_set()) [0x200002b15:0x98a:0x0] lo_layout_invalid=1 00020000:00000001:2.0:1775773861.344594:0:207610:0:(lov_object.c:1500:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.344595:0:207610:0:(cl_object.c:359:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.344596:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.344597:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.344599:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.344600:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.344602:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.344603:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.344605:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.344606:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.344607:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.344609:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.344610:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.344611:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.344613:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.344614:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.344615:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.344616:0:207610:0:(file.c:6773:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.344617:0:207610:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:2.0:1775773861.344619:0:207610:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b15:0x98a:0x0](ffff8f621efb0908) invalid 00000020:00000001:2.0:1775773861.344620:0:207610:0:(obd_sysfs.c:903:encode_fn_len()) Process filename at ffffaf0701cebca0 00000080:00002000:2.0:1775773861.344621:0:207610:0:(llite_internal.h:2024:d_lustre_invalidate()) invalidate dentry f.98 (ffff8f62191c5000) parent ffff8f6202184798 inode ffff8f621efb0908 refc 0 00000080:00000001:2.0:1775773861.344624:0:207610:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:2.0:1775773861.344626:0:207610:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:2.0:1775773861.344628:0:207610:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:2.0:1775773861.344630:0:207610:0:(namei.c:402:ll_lock_cancel_bits()) Process leaving 00000080:00000001:2.0:1775773861.344630:0:207610:0:(namei.c:536:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775773861.344632:0:207610:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1775773861.344633:0:207610:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1775773861.344634:0:207610:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1775773861.344635:0:207610:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1775773861.344636:0:207610:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775773861.344638:0:207610:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8f621839e600 with handle 0xf74d1861c614800 from hash 00010000:00000001:2.0:1775773861.344639:0:207610:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1775773861.344640:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1775773861.344641:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.344642:0:207610:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1775773861.344643:0:207610:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1775773861.344644:0:207610:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:2.0:1775773861.344646:0:207610:0:(ldlm_request.c:1386:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1775773861.344648:0:207610:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226128000/0xf74d1861c6147f9 lrc: 2/0,0 mode: PW/PW res: [0x200002b15:0x989:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x800028400000000 nid: local remote: 0xd88660ed39b2cc7e expref: -99 pid: 214086 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00000001:2.0:1775773861.344653:0:207610:0:(namei.c:472:ll_md_blocking_ast()) Process entered 00000080:00000001:2.0:1775773861.344654:0:207610:0:(namei.c:239:ll_lock_cancel_bits()) Process entered 00000080:00000001:2.0:1775773861.344655:0:207610:0:(file.c:6026:ll_have_md_lock()) Process entered 00000080:00000040:2.0:1775773861.344656:0:207610:0:(file.c:6032:ll_have_md_lock()) trying to match res [0x200002b15:0x989:0x0] mode (null) 00000002:00000001:2.0:1775773861.344657:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.344658:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.344660:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960a00 count: 2 00010000:00000001:2.0:1775773861.344661:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.344663:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.344664:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.344664:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.344666:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960a00 count: 2 00010000:00000001:2.0:1775773861.344667:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.344668:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.344669:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.344670:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.344671:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960a00 count: 2 00010000:00000001:2.0:1775773861.344672:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.344673:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.344674:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.344674:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.344675:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960a00 count: 2 00010000:00000001:2.0:1775773861.344676:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.344677:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.344678:0:207610:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:2.0:1775773861.344679:0:207610:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:2.0:1775773861.344680:0:207610:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff8f6233960a00 count: 2 00010000:00000001:2.0:1775773861.344681:0:207610:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:2.0:1775773861.344682:0:207610:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.344683:0:207610:0:(file.c:6054:ll_have_md_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.344684:0:207610:0:(namei.c:211:ll_dom_lock_cancel()) Process entered 00000020:00000001:2.0:1775773861.344684:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.344686:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.344687:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.344689:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.344691:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.344692:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.344694:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.344695:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.344697:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.344698:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.344700:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.344701:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.344703:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.344704:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.344706:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.344707:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.344708:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.344709:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00020000:00000002:2.0:1775773861.344710:0:207610:0:(lov_object.c:1217:lov_conf_freeze()) To take share lov(ffff8f622e196ba0) owner 0000000000000000/ffff8f62012d8000 00020000:00000001:2.0:1775773861.344712:0:207610:0:(lov_object.c:1110:lov_flush_composite()) Process entered 00000020:00000001:2.0:1775773861.344713:0:207610:0:(cl_object.c:488:cl_object_flush()) Process entered 00000002:00000001:2.0:1775773861.344714:0:207610:0:(mdc_dev.c:306:mdc_dlm_canceling()) Process entered 00000002:00000001:2.0:1775773861.344715:0:207610:0:(mdc_dev.c:255:mdc_lock_flush()) Process entered 00000008:00000001:2.0:1775773861.344716:0:207610:0:(osc_cache.c:3265:osc_cache_writeback_range()) Process entered 00000008:00000001:2.0:1775773861.344717:0:207610:0:(osc_cache.c:3200:osc_cache_wait_range()) Process entered 00000008:00000020:2.0:1775773861.344718:0:207610:0:(osc_cache.c:3237:osc_cache_wait_range()) obj ffff8f623476ee00 ready 0|-|- wr 0|-|- rd 0|- sync file range. 00000008:00000001:2.0:1775773861.344720:0:207610:0:(osc_cache.c:3238:osc_cache_wait_range()) Process leaving (rc=0 : 0 : 0) 00000008:00000020:2.0:1775773861.344721:0:207610:0:(osc_cache.c:3397:osc_cache_writeback_range()) obj ffff8f623476ee00 ready 0|-|- wr 0|-|- rd 0|- pageout [0, 18446744073709551615] npages 0: rc=0. 00000008:00000001:2.0:1775773861.344724:0:207610:0:(osc_cache.c:3412:osc_cache_writeback_range()) Process leaving (rc=0 : 0 : 0) 00000002:00000020:2.0:1775773861.344725:0:207610:0:(mdc_dev.c:262:mdc_lock_flush()) object ffff8f623476ee00: [0 -> 18446744073709551615] 0 pages were written back. 00000002:00000001:2.0:1775773861.344726:0:207610:0:(mdc_dev.c:228:mdc_lock_discard_pages()) Process entered 00000020:00000001:2.0:1775773861.344727:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476ee00 -> ffff8f622c2e0dc8 00000020:00000001:2.0:1775773861.344728:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f622c2e0dc8 -> ffff8f622c2e0dc8 00000020:00000001:2.0:1775773861.344729:0:207610:0:(cl_io.c:125:__cl_io_init()) Process entered 00000080:00000001:2.0:1775773861.344729:0:207610:0:(vvp_io.c:1812:vvp_io_init()) Process entered 00000080:00200000:2.0:1775773861.344730:0:207610:0:(vvp_io.c:1817:vvp_io_init()) [0x200002b15:0x989:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000020:00000001:2.0:1775773861.344732:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.344733:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00000080:00000001:2.0:1775773861.344734:0:207610:0:(vvp_io.c:1889:vvp_io_init()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.344735:0:207610:0:(lov_object.c:1554:lov_io_init()) [0x200002b15:0x989:0x0]io ffff8f6205260bc8 type 8 ignore/verify layout 1/0 00020000:00000001:2.0:1775773861.344737:0:207610:0:(lov_io.c:1952:lov_io_init_composite()) Process entered 00020000:00000001:2.0:1775773861.344737:0:207610:0:(lov_io.c:495:lov_io_slice_init()) Process entered 00020000:00000001:2.0:1775773861.344739:0:207610:0:(lov_io.c:599:lov_io_slice_init()) Process leaving via out (rc=0 : 0 : 0x0) 00020000:00000001:2.0:1775773861.344740:0:207610:0:(lov_io.c:203:lov_io_subio_init()) Process entered 00020000:00000001:2.0:1775773861.344740:0:207610:0:(lov_io.c:211:lov_io_subio_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.344741:0:207610:0:(cl_io.c:898:cl_io_slice_add()) Process entered 00000020:00000001:2.0:1775773861.344742:0:207610:0:(cl_io.c:904:cl_io_slice_add()) Process leaving 00020000:00000001:2.0:1775773861.344742:0:207610:0:(lov_io.c:1965:lov_io_init_composite()) Process leaving 00000020:00000001:2.0:1775773861.344743:0:207610:0:(cl_io.c:142:__cl_io_init()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775773861.344744:0:207610:0:(osc_cache.c:3447:osc_page_gang_lookup()) Process entered 00000008:00000001:2.0:1775773861.344745:0:207610:0:(osc_cache.c:3524:osc_page_gang_lookup()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775773861.344746:0:207610:0:(cl_io.c:71:cl_io_fini()) Process entered 00020000:00000001:2.0:1775773861.344747:0:207610:0:(lov_io.c:675:lov_io_fini()) Process entered 00020000:00000001:2.0:1775773861.344748:0:207610:0:(lov_io.c:694:lov_io_fini()) Process leaving 00000080:00000001:2.0:1775773861.344748:0:207610:0:(vvp_io.c:280:vvp_io_fini()) Process entered 00000080:00200000:2.0:1775773861.344749:0:207610:0:(vvp_io.c:288:vvp_io_fini()) [0x200002b15:0x989:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00000001:2.0:1775773861.344751:0:207610:0:(vvp_io.c:386:vvp_io_fini()) Process leaving 00000020:00000001:2.0:1775773861.344751:0:207610:0:(cl_io.c:112:cl_io_fini()) Process leaving 00000002:00000001:2.0:1775773861.344752:0:207610:0:(mdc_dev.c:245:mdc_lock_discard_pages()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.344753:0:207610:0:(mdc_dev.c:273:mdc_lock_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.344754:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476ee00 -> ffff8f622c2e0dc8 00000020:00000001:2.0:1775773861.344755:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476ee00 -> ffff8f622c2e0dc8 00000020:00000001:2.0:1775773861.344756:0:207610:0:(cl_object.c:256:cl_object_attr_update()) Process entered 00000008:00000020:2.0:1775773861.344757:0:207610:0:(osc_object.c:161:osc_attr_update()) set kms from 131072to 0 00020000:00000001:2.0:1775773861.344758:0:207610:0:(lovsub_object.c:98:lovsub_attr_update()) Process entered 00020000:00000001:2.0:1775773861.344759:0:207610:0:(lovsub_object.c:100:lovsub_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.344760:0:207610:0:(cl_object.c:269:cl_object_attr_update()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.344761:0:207610:0:(cl_object.c:151:cl_object_top()) ffff8f623476ee00 -> ffff8f622c2e0dc8 00000002:00000001:2.0:1775773861.344763:0:207610:0:(mdc_dev.c:346:mdc_dlm_canceling()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:2.0:1775773861.344764:0:207610:0:(mdc_dev.c:1535:mdc_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.344765:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1775773861.344766:0:207610:0:(lov_object.c:1119:lov_flush_composite()) Process leaving (rc=0 : 0 : 0) 00020000:00000002:2.0:1775773861.344767:0:207610:0:(lov_object.c:1225:lov_conf_thaw()) To release share lov(ffff8f622e196ba0) owner 0000000000000000/ffff8f62012d8000 00000020:00000001:2.0:1775773861.344768:0:207610:0:(cl_object.c:497:cl_object_flush()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.344769:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.344771:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.344773:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.344774:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.344776:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.344777:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.344779:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.344780:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.344782:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4400. 00000008:00000010:2.0:1775773861.344783:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.344784:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.344786:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.344787:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.344789:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.344790:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.344792:0:207610:0:(namei.c:229:ll_dom_lock_cancel()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.344793:0:207610:0:(file.c:6731:ll_layout_conf()) Process entered 00000020:00000001:2.0:1775773861.344793:0:207610:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1775773861.344794:0:207610:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff8f6209157e10. 00000020:00000010:2.0:1775773861.344796:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c4800. 00000020:00000010:2.0:1775773861.344797:0:207610:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.344799:0:207610:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.344801:0:207610:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.344803:0:207610:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.344805:0:207610:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.344808:0:207610:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.344809:0:207610:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.344812:0:207610:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff8f62066c5200. 00000008:00000010:2.0:1775773861.344814:0:207610:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.344817:0:207610:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.344819:0:207610:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.344822:0:207610:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff8f62052b25a0. 00000020:00000001:2.0:1775773861.344824:0:207610:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446620249954811416 : -123823754740200 : ffff8f6209157e18) 00000020:00001000:2.0:1775773861.344826:0:207610:0:(cl_object.c:831:cl_env_get()) 1@ffff8f6209157e10 00000020:00000001:2.0:1775773861.344828:0:207610:0:(cl_object.c:351:cl_conf_set()) Process entered 00000080:00200000:2.0:1775773861.344829:0:207610:0:(vvp_object.c:117:vvp_conf_set()) [0x200002b15:0x989:0x0]: losing layout lock 00000080:00000001:2.0:1775773861.344831:0:207610:0:(pcc.c:2213:pcc_layout_invalidate()) Process entered 00000080:00000001:2.0:1775773861.344833:0:207610:0:(pcc.c:2225:pcc_layout_invalidate()) Process leaving 00020000:00000001:2.0:1775773861.344835:0:207610:0:(lov_object.c:1412:lov_conf_set()) Process entered 00020000:00000001:2.0:1775773861.344836:0:207610:0:(lov_object.c:1426:lov_conf_set()) Process leaving via out_lsm (rc=0 : 0 : 0x0) 00020000:00000002:2.0:1775773861.344838:0:207610:0:(lov_object.c:1499:lov_conf_set()) [0x200002b15:0x989:0x0] lo_layout_invalid=1 00020000:00000001:2.0:1775773861.344840:0:207610:0:(lov_object.c:1500:lov_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.344842:0:207610:0:(cl_object.c:359:cl_conf_set()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1775773861.344844:0:207610:0:(cl_object.c:925:cl_env_put()) 1@ffff8f6209157e10 00000020:00000010:2.0:1775773861.344846:0:207610:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff8f62066c4400. 00000020:00000010:2.0:1775773861.344848:0:207610:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff8f6206b1d720. 00000008:00000010:2.0:1775773861.344851:0:207610:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff8f6205260b28. 00020000:00000010:2.0:1775773861.344855:0:207610:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff8f622c2e4240. 00008000:00000010:2.0:1775773861.344858:0:207610:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff8f6211958e20. 00000080:00000010:2.0:1775773861.344861:0:207610:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff8f6211642348. 00000080:00000010:2.0:1775773861.344864:0:207610:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff8f6209153230. 00000020:00000010:2.0:1775773861.344868:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c4800. 00000008:00000010:2.0:1775773861.344871:0:207610:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff8f6209e4a580. 00020000:00000010:2.0:1775773861.344874:0:207610:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff8f62091906c0. 00008000:00000010:2.0:1775773861.344877:0:207610:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff8f6209199310. 00000080:00000010:2.0:1775773861.344880:0:207610:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff8f62052b25a0. 00000020:00000010:2.0:1775773861.344883:0:207610:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff8f62066c5200. 00000020:00000010:2.0:1775773861.344886:0:207610:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff8f6209157e10. 00000080:00000001:2.0:1775773861.344888:0:207610:0:(file.c:6773:ll_layout_conf()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1775773861.344890:0:207610:0:(dcache.c:195:ll_prune_aliases()) Process entered 00000080:00000002:2.0:1775773861.344891:0:207610:0:(dcache.c:200:ll_prune_aliases()) marking dentries for inode [0x200002b15:0x989:0x0](ffff8f621efbaa08) invalid 00000020:00000001:2.0:1775773861.344894:0:207610:0:(obd_sysfs.c:903:encode_fn_len()) Process filename at ffffaf0701cebca0 00000080:00002000:2.0:1775773861.344896:0:207610:0:(llite_internal.h:2024:d_lustre_invalidate()) invalidate dentry f.97 (ffff8f62191fa000) parent ffff8f6202184798 inode ffff8f621efbaa08 refc 0 00000080:00000001:2.0:1775773861.344900:0:207610:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:2.0:1775773861.344905:0:207610:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:2.0:1775773861.344908:0:207610:0:(dcache.c:120:ll_ddelete()) Process entered 00000020:00000001:2.0:1775773861.344910:0:207610:0:(obd_sysfs.c:903:encode_fn_len()) Process filename at ffffaf0701cebc10 00000080:00002000:2.0:1775773861.344912:0:207610:0:(dcache.c:127:ll_ddelete()) deleting dentry 11 (ffff8f6202184798, parent ffff8f62021900d8, inode ffff8f62268600c8) hashed, 00000080:00000001:2.0:1775773861.344916:0:207610:0:(dcache.c:130:ll_ddelete()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:2.0:1775773861.344919:0:207610:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:2.0:1775773861.344921:0:207610:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:2.0:1775773861.344923:0:207610:0:(dcache.c:209:ll_prune_aliases()) Process leaving 00000080:00000001:2.0:1775773861.344924:0:207610:0:(namei.c:402:ll_lock_cancel_bits()) Process leaving 00000080:00000001:2.0:1775773861.344925:0:207610:0:(namei.c:536:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775773861.344927:0:207610:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1775773861.344929:0:207610:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1775773861.344930:0:207610:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1775773861.344932:0:207610:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1775773861.344934:0:207610:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775773861.344936:0:207610:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff8f6226128000 with handle 0xf74d1861c6147f9 from hash 00010000:00000001:2.0:1775773861.344937:0:207610:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1775773861.344938:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1775773861.344939:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.344940:0:207610:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1775773861.344942:0:207610:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1775773861.344943:0:207610:0:(ldlm_request.c:1422:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:2.0:1775773861.344945:0:207610:0:(ldlm_request.c:1880:ldlm_cli_cancel_list_local()) Process leaving (rc=127 : 127 : 7f) 00010000:00000001:2.0:1775773861.344948:0:207610:0:(ldlm_request.c:2444:ldlm_cli_cancel_list()) Process entered 00010000:00000001:2.0:1775773861.344950:0:207610:0:(ldlm_request.c:1519:ldlm_cli_cancel_req()) Process entered 00000100:00000010:2.0:1775773861.344957:0:207610:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448000. 00000020:00000040:2.0:1775773861.344960:0:207610:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000001:2.0:1775773861.344964:0:207610:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773861.344966:0:207610:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773861.344970:0:207610:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773861.344977:0:207610:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 2048 at ffff8f622c04d000. 00000100:00000001:2.0:1775773861.344982:0:207610:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1775773861.344984:0:207610:0:(ldlm_request.c:1460:ldlm_cancel_pack()) Process entered 00010000:00010000:2.0:1775773861.344989:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c0200/0xf74d1861c614b8e lrc: 1/0,0 mode: --/PW res: [0x200002b15:0xa07:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2de2f expref: -99 pid: 214220 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.344994:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c1a00/0xf74d1861c614b87 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0xa06:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2de0c expref: -99 pid: 214219 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345000:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233db6c00/0xf74d1861c614b80 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0xa05:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dde9 expref: -99 pid: 214218 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345008:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c1000/0xf74d1861c614b79 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0xa04:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2ddc6 expref: -99 pid: 214217 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345013:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c1e00/0xf74d1861c614b72 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0xa03:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dda3 expref: -99 pid: 214216 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345017:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c1200/0xf74d1861c614b6b lrc: 1/0,0 mode: --/PW res: [0x200002b15:0xa02:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dd80 expref: -99 pid: 214215 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345022:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c0400/0xf74d1861c614b64 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0xa01:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dd5d expref: -99 pid: 214214 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345026:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c0800/0xf74d1861c614b5d lrc: 1/0,0 mode: --/PW res: [0x200002b15:0xa00:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dd3a expref: -99 pid: 214213 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345031:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c1800/0xf74d1861c614b56 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9ff:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dd17 expref: -99 pid: 214212 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345035:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c0600/0xf74d1861c614b4f lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9fe:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dcf4 expref: -99 pid: 214211 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345039:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622cc0d200/0xf74d1861c614b48 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9fd:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dcd1 expref: -99 pid: 214210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345044:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622d8f6800/0xf74d1861c614b41 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9fc:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dcae expref: -99 pid: 214209 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345048:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622cc0d600/0xf74d1861c614b3a lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9fb:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dc8b expref: -99 pid: 214208 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345054:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c0000/0xf74d1861c614b33 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9fa:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dc68 expref: -99 pid: 214207 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345058:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c0a00/0xf74d1861c614b2c lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9f9:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dc45 expref: -99 pid: 214206 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345062:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c1400/0xf74d1861c614b25 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9f8:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dc22 expref: -99 pid: 214205 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345067:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5fc00/0xf74d1861c614b1e lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9f7:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dbff expref: -99 pid: 214204 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345071:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5ea00/0xf74d1861c614b17 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9f6:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dbdc expref: -99 pid: 214203 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345075:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622cc0da00/0xf74d1861c614b10 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9f5:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dbb9 expref: -99 pid: 214202 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345080:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622cc0d000/0xf74d1861c614b09 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9f4:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2db96 expref: -99 pid: 214201 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345084:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476d800/0xf74d1861c614b02 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9f3:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2db73 expref: -99 pid: 214200 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345088:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5f400/0xf74d1861c614afb lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9f2:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2db50 expref: -99 pid: 214199 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345092:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233db7600/0xf74d1861c614af4 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9f1:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2db2d expref: -99 pid: 214198 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345098:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233db6e00/0xf74d1861c614aed lrc: 1/0,0 mode: --/PR res: [0x200002b15:0x9f0:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd88660ed39b2db11 expref: -99 pid: 214198 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775773861.345103:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476ca00/0xf74d1861c614ac3 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9ef:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2da70 expref: -99 pid: 214189 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345107:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5ec00/0xf74d1861c614abc lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9ee:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2da4d expref: -99 pid: 214188 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345111:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5f600/0xf74d1861c614ab5 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9ed:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2da2a expref: -99 pid: 214187 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345115:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5ee00/0xf74d1861c614aae lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9ec:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2da07 expref: -99 pid: 214186 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345119:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233db6200/0xf74d1861c614aa7 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9eb:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d9e4 expref: -99 pid: 214185 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345123:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5e200/0xf74d1861c614aa0 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9ea:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d9c1 expref: -99 pid: 214184 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345128:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233db7a00/0xf74d1861c614a99 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9e9:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d99e expref: -99 pid: 214183 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345132:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5fa00/0xf74d1861c614a92 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9e8:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d97b expref: -99 pid: 214182 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345152:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e298400/0xf74d1861c614a8b lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9e7:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d958 expref: -99 pid: 214181 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345158:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e298800/0xf74d1861c614a84 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9e6:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d935 expref: -99 pid: 214180 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345162:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5f000/0xf74d1861c614a7d lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9e5:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d912 expref: -99 pid: 214179 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345167:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e299800/0xf74d1861c614a76 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9e4:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d8ef expref: -99 pid: 214178 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345171:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5fe00/0xf74d1861c614a6f lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9e3:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d8cc expref: -99 pid: 214177 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345175:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476cc00/0xf74d1861c614a68 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9e2:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d8a9 expref: -99 pid: 214176 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345179:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5f200/0xf74d1861c614a61 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9e1:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d886 expref: -99 pid: 214175 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345183:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5e400/0xf74d1861c614a5a lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9e0:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d863 expref: -99 pid: 214174 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345187:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5e800/0xf74d1861c614a53 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9df:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d840 expref: -99 pid: 214173 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345191:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5f800/0xf74d1861c614a4c lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9de:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d81d expref: -99 pid: 214172 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345196:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5e600/0xf74d1861c614a45 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9dd:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d7fa expref: -99 pid: 214171 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345201:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5e000/0xf74d1861c614a3e lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9dc:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d7d7 expref: -99 pid: 214170 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345205:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476d600/0xf74d1861c614a37 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9db:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d7b4 expref: -99 pid: 214169 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345210:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f55200/0xf74d1861c614a30 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9da:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d791 expref: -99 pid: 214168 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345214:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476d400/0xf74d1861c614a29 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9d9:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d76e expref: -99 pid: 214167 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345218:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476c800/0xf74d1861c614a22 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9d8:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d74b expref: -99 pid: 214166 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345222:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f54400/0xf74d1861c614a1b lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9d7:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d728 expref: -99 pid: 214165 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345226:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f54800/0xf74d1861c614a14 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9d6:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d705 expref: -99 pid: 214164 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345231:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f55800/0xf74d1861c614a0d lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9d5:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d6e2 expref: -99 pid: 214163 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345235:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e298600/0xf74d1861c614a06 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9d4:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d6bf expref: -99 pid: 214162 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345239:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f54600/0xf74d1861c6149ff lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9d3:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d69c expref: -99 pid: 214161 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345245:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e298000/0xf74d1861c6149f8 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9d2:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d679 expref: -99 pid: 214160 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345249:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f54000/0xf74d1861c6149f1 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9d1:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d656 expref: -99 pid: 214159 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345253:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f54a00/0xf74d1861c6149ea lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9d0:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d633 expref: -99 pid: 214158 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345258:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f55400/0xf74d1861c6149e3 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9cf:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d610 expref: -99 pid: 214157 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345262:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e298a00/0xf74d1861c6149dc lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9ce:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d5ed expref: -99 pid: 214156 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345266:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622d8f7000/0xf74d1861c6149d5 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9cd:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d5ca expref: -99 pid: 214154 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345270:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476ce00/0xf74d1861c6149ce lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9cc:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d5a7 expref: -99 pid: 214153 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345274:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f55c00/0xf74d1861c6149c7 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9cb:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d584 expref: -99 pid: 214152 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345278:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f54c00/0xf74d1861c6149c0 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9ca:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d561 expref: -99 pid: 214151 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345282:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622d8f6400/0xf74d1861c6149b9 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9c9:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d53e expref: -99 pid: 214150 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345287:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f55600/0xf74d1861c6149b2 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9c8:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d51b expref: -99 pid: 214149 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345292:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476d200/0xf74d1861c6149ab lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9c7:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d4f8 expref: -99 pid: 214148 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345297:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f54e00/0xf74d1861c6149a4 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9c6:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d4d5 expref: -99 pid: 214147 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345301:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476c200/0xf74d1861c61499d lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9c5:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d4b2 expref: -99 pid: 214146 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345305:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f54200/0xf74d1861c614996 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9c4:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d48f expref: -99 pid: 214145 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345309:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e299400/0xf74d1861c61498f lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9c3:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d46c expref: -99 pid: 214144 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345313:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622d8f7a00/0xf74d1861c614988 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9c2:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d449 expref: -99 pid: 214143 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345318:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622d8f7600/0xf74d1861c614981 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9c1:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d426 expref: -99 pid: 214142 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345322:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622d8f6c00/0xf74d1861c61497a lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9c0:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d403 expref: -99 pid: 214141 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345326:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e299c00/0xf74d1861c614973 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9bf:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d3e0 expref: -99 pid: 214140 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345331:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f55a00/0xf74d1861c61496c lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9be:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d3bd expref: -99 pid: 214139 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345336:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f55000/0xf74d1861c614965 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9bd:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d39a expref: -99 pid: 214138 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345340:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f55e00/0xf74d1861c61495e lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9bc:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d377 expref: -99 pid: 214137 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345344:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622d8f6a00/0xf74d1861c614957 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9bb:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d354 expref: -99 pid: 214136 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345348:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c0400/0xf74d1861c614950 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9ba:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d331 expref: -99 pid: 214135 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345352:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e298c00/0xf74d1861c614949 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9b9:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d30e expref: -99 pid: 214134 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345357:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c0800/0xf74d1861c614942 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9b8:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d2eb expref: -99 pid: 214133 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345361:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e299600/0xf74d1861c61493b lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9b7:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d2c8 expref: -99 pid: 214132 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345365:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c1800/0xf74d1861c614934 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9b6:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d2a5 expref: -99 pid: 214131 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345369:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62265d7000/0xf74d1861c61492d lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9b5:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d282 expref: -99 pid: 214130 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345374:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c0600/0xf74d1861c614926 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9b4:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d25f expref: -99 pid: 214129 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345379:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c0000/0xf74d1861c61491f lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9b3:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d23c expref: -99 pid: 214128 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345383:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e298e00/0xf74d1861c614918 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9b2:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d219 expref: -99 pid: 214127 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345387:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e298200/0xf74d1861c614911 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9b1:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d1f6 expref: -99 pid: 214126 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345391:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c0a00/0xf74d1861c61490a lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9b0:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d1d3 expref: -99 pid: 214125 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345395:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c1400/0xf74d1861c614903 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9af:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d1b0 expref: -99 pid: 214124 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345399:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e299a00/0xf74d1861c6148fc lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9ae:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d18d expref: -99 pid: 214123 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345403:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e299000/0xf74d1861c6148f5 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9ad:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d16a expref: -99 pid: 214122 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345407:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c1c00/0xf74d1861c6148ee lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9ac:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d147 expref: -99 pid: 214121 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345411:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c0c00/0xf74d1861c6148e7 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9ab:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d124 expref: -99 pid: 214120 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345416:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62265d6e00/0xf74d1861c6148e0 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9aa:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d101 expref: -99 pid: 214119 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345421:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c1600/0xf74d1861c6148d9 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9a9:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d0de expref: -99 pid: 214118 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345425:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c0e00/0xf74d1861c6148d2 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9a8:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d0bb expref: -99 pid: 214117 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345429:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476c600/0xf74d1861c6148cb lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9a7:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d098 expref: -99 pid: 214116 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345433:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c0200/0xf74d1861c6148c4 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9a6:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d075 expref: -99 pid: 214115 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345437:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e299e00/0xf74d1861c6148bd lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9a5:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d052 expref: -99 pid: 214114 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345441:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62265d6400/0xf74d1861c6148b6 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9a4:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d02f expref: -99 pid: 214113 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345445:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476da00/0xf74d1861c6148af lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9a3:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d00c expref: -99 pid: 214112 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345449:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476de00/0xf74d1861c6148a8 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9a2:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cfe9 expref: -99 pid: 214111 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345453:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476c400/0xf74d1861c6148a1 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9a1:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cfc6 expref: -99 pid: 214110 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345459:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e299200/0xf74d1861c61489a lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x9a0:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cfa3 expref: -99 pid: 214109 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345464:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c1a00/0xf74d1861c614893 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x99f:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cf80 expref: -99 pid: 214108 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345468:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c1000/0xf74d1861c61488c lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x99e:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cf5d expref: -99 pid: 214107 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345484:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c1e00/0xf74d1861c614885 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x99d:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cf3a expref: -99 pid: 214106 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345488:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c1200/0xf74d1861c61487e lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x99c:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cf17 expref: -99 pid: 214105 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345493:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226128e00/0xf74d1861c614877 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x99b:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cef4 expref: -99 pid: 214104 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345497:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622c1ec200/0xf74d1861c614870 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x99a:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2ced1 expref: -99 pid: 214103 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345501:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226128200/0xf74d1861c614869 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x999:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2ceae expref: -99 pid: 214102 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345505:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226129a00/0xf74d1861c614862 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x998:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2ce8b expref: -99 pid: 214101 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345509:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226129000/0xf74d1861c61485b lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x997:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2ce68 expref: -99 pid: 214100 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345514:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226129e00/0xf74d1861c614854 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x996:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2ce45 expref: -99 pid: 214099 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345518:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f621839e800/0xf74d1861c61484d lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x995:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2ce22 expref: -99 pid: 214098 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345522:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62265d6200/0xf74d1861c614846 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x994:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cdff expref: -99 pid: 214097 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345526:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226129200/0xf74d1861c61483f lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x993:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cddc expref: -99 pid: 214096 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345530:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f621839f800/0xf74d1861c614838 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x992:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cdb9 expref: -99 pid: 214095 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345534:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226128400/0xf74d1861c614831 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x991:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cd96 expref: -99 pid: 214094 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345538:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476dc00/0xf74d1861c61482a lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x990:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cd73 expref: -99 pid: 214093 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345542:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226128800/0xf74d1861c614823 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x98f:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cd50 expref: -99 pid: 214092 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345546:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62265d7a00/0xf74d1861c61481c lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x98e:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cd2d expref: -99 pid: 214091 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345549:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226129800/0xf74d1861c614815 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x98d:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cd0a expref: -99 pid: 214090 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345554:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476d000/0xf74d1861c61480e lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x98c:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cce7 expref: -99 pid: 214089 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345559:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226128600/0xf74d1861c614807 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x98b:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2ccc4 expref: -99 pid: 214088 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345563:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f621839e600/0xf74d1861c614800 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x98a:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cca1 expref: -99 pid: 214087 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345567:0:207610:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226128000/0xf74d1861c6147f9 lrc: 1/0,0 mode: --/PW res: [0x200002b15:0x989:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cc7e expref: -99 pid: 214086 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775773861.345571:0:207610:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 127 locks packed 00010000:00000001:2.0:1775773861.345572:0:207610:0:(ldlm_request.c:1492:ldlm_cancel_pack()) Process leaving (rc=127 : 127 : 7f) 00000100:00000001:2.0:1775773861.345576:0:207610:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773861.345580:0:207610:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773861.345591:0:207610:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043ee9227] to pc [ptlrpcd_01_01+1] req@ffff8f6233448000 x1862022710221312/t0(0) o103->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:17/18 lens 1328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1775773861.345613:0:207610:0:(ldlm_request.c:1587:ldlm_cli_cancel_req()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1775773861.345615:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.345616:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c0200/0xf74d1861c614b8e lrc: 0/0,0 mode: --/PW res: [0x200002b15:0xa07:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2de2f expref: -99 pid: 214220 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.345622:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f62062cb700. 00010000:00000040:2.0:1775773861.345625:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6206b66d00 count: 0 00010000:00000010:2.0:1775773861.345629:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f62062cbd80. 00010000:00000001:2.0:1775773861.345636:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.345638:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.345639:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c1a00/0xf74d1861c614b87 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0xa06:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2de0c expref: -99 pid: 214219 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.345645:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f62062cbd00. 00010000:00000040:2.0:1775773861.345648:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6206b66400 count: 0 00010000:00000010:2.0:1775773861.345653:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f62062cb800. 00010000:00000001:2.0:1775773861.345659:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.345661:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.345663:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233db6c00/0xf74d1861c614b80 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0xa05:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dde9 expref: -99 pid: 214218 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.345677:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6226713d80. 00010000:00000040:2.0:1775773861.345683:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0afa00 count: 0 00010000:00000010:2.0:1775773861.345688:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6226713c80. 00010000:00000001:2.0:1775773861.345693:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.345695:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.345697:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c1000/0xf74d1861c614b79 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0xa04:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2ddc6 expref: -99 pid: 214217 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.345709:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f62062cb680. 00010000:00000040:2.0:1775773861.345713:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6206b66800 count: 0 00010000:00000010:2.0:1775773861.345716:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f62062cb000. 00010000:00000001:2.0:1775773861.345721:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.345724:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.345726:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c1e00/0xf74d1861c614b72 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0xa03:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dda3 expref: -99 pid: 214216 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.345737:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f62062cb900. 00010000:00000040:2.0:1775773861.345740:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6206b66e00 count: 0 00010000:00000010:2.0:1775773861.345746:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f62062cb780. 00010000:00000001:2.0:1775773861.345751:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.345753:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.345756:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c1200/0xf74d1861c614b6b lrc: 0/0,0 mode: --/PW res: [0x200002b15:0xa02:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dd80 expref: -99 pid: 214215 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.345767:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f62062cbe80. 00010000:00000040:2.0:1775773861.345770:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6206b66900 count: 0 00010000:00000010:2.0:1775773861.345773:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f62062cba80. 00010000:00000001:2.0:1775773861.345780:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.345781:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.345783:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c0400/0xf74d1861c614b64 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0xa01:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dd5d expref: -99 pid: 214214 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.345792:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f62062cb380. 00010000:00000040:2.0:1775773861.345795:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f620618fd00 count: 0 00010000:00000010:2.0:1775773861.345798:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f62062cbc00. 00010000:00000001:2.0:1775773861.345804:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.345806:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.345809:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c0800/0xf74d1861c614b5d lrc: 0/0,0 mode: --/PW res: [0x200002b15:0xa00:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dd3a expref: -99 pid: 214213 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.345817:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f62062cbb00. 00010000:00000040:2.0:1775773861.345821:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f620618f100 count: 0 00010000:00000010:2.0:1775773861.345825:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f62062cb880. 00010000:00000001:2.0:1775773861.345829:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.345832:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.345835:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c1800/0xf74d1861c614b56 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9ff:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dd17 expref: -99 pid: 214212 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.345845:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f62062cbe00. 00010000:00000040:2.0:1775773861.345848:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f620618f600 count: 0 00010000:00000010:2.0:1775773861.345852:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f62062cbb80. 00010000:00000001:2.0:1775773861.345856:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.345857:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.345858:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c0600/0xf74d1861c614b4f lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9fe:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dcf4 expref: -99 pid: 214211 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.345865:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f62062cba00. 00010000:00000040:2.0:1775773861.345868:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f620618ff00 count: 0 00010000:00000010:2.0:1775773861.345871:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f62062cb500. 00010000:00000001:2.0:1775773861.345876:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.345878:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.345879:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622cc0d200/0xf74d1861c614b48 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9fd:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dcd1 expref: -99 pid: 214210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.345889:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f62062cb400. 00010000:00000040:2.0:1775773861.345892:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f620618f000 count: 0 00010000:00000010:2.0:1775773861.345895:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f62062cbf80. 00010000:00000001:2.0:1775773861.345899:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.345901:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.345905:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622d8f6800/0xf74d1861c614b41 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9fc:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dcae expref: -99 pid: 214209 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.345914:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6226641380. 00010000:00000040:2.0:1775773861.345917:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622ce91400 count: 0 00010000:00000010:2.0:1775773861.345920:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622e182000. 00010000:00000001:2.0:1775773861.345927:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.345931:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.345934:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622cc0d600/0xf74d1861c614b3a lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9fb:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dc8b expref: -99 pid: 214208 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.345943:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622e182e00. 00010000:00000040:2.0:1775773861.345945:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622ce91800 count: 0 00010000:00000010:2.0:1775773861.345946:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622e182180. 00010000:00000001:2.0:1775773861.345950:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.345951:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.345953:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c0000/0xf74d1861c614b33 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9fa:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dc68 expref: -99 pid: 214207 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.345957:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f62062cbf00. 00010000:00000040:2.0:1775773861.345959:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f620618f500 count: 0 00010000:00000010:2.0:1775773861.345960:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f62062cb080. 00010000:00000001:2.0:1775773861.345963:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.345963:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.345965:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c0a00/0xf74d1861c614b2c lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9f9:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dc45 expref: -99 pid: 214206 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.345968:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6226713800. 00010000:00000040:2.0:1775773861.345969:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0af100 count: 0 00010000:00000010:2.0:1775773861.345971:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6226713200. 00010000:00000001:2.0:1775773861.345973:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.345974:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.345975:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62264c1400/0xf74d1861c614b25 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9f8:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dc22 expref: -99 pid: 214205 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.345979:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f62062cb580. 00010000:00000040:2.0:1775773861.345980:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f620618fb00 count: 0 00010000:00000010:2.0:1775773861.345981:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f62062cb180. 00010000:00000001:2.0:1775773861.345984:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.345984:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.345986:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5fc00/0xf74d1861c614b1e lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9f7:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dbff expref: -99 pid: 214204 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.345989:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f62062cb980. 00010000:00000040:2.0:1775773861.345990:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f620618fc00 count: 0 00010000:00000010:2.0:1775773861.345992:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f62062cb100. 00010000:00000001:2.0:1775773861.345994:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.345995:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.345996:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5ea00/0xf74d1861c614b17 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9f6:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dbdc expref: -99 pid: 214203 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346001:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6226713000. 00010000:00000040:2.0:1775773861.346002:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0af600 count: 0 00010000:00000010:2.0:1775773861.346004:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6226713900. 00010000:00000001:2.0:1775773861.346007:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346007:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346009:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622cc0da00/0xf74d1861c614b10 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9f5:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2dbb9 expref: -99 pid: 214202 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346012:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622586cd00. 00010000:00000040:2.0:1775773861.346014:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f620618f300 count: 0 00010000:00000010:2.0:1775773861.346016:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622586cb00. 00010000:00000001:2.0:1775773861.346018:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346019:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346021:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622cc0d000/0xf74d1861c614b09 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9f4:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2db96 expref: -99 pid: 214201 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346024:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233b37600. 00010000:00000040:2.0:1775773861.346026:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f621e49d600 count: 0 00010000:00000010:2.0:1775773861.346028:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233b37500. 00010000:00000001:2.0:1775773861.346030:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346031:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346032:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476d800/0xf74d1861c614b02 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9f3:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2db73 expref: -99 pid: 214200 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346036:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233b37680. 00010000:00000040:2.0:1775773861.346037:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f621e49df00 count: 0 00010000:00000010:2.0:1775773861.346038:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233b37200. 00010000:00000001:2.0:1775773861.346041:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346042:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346043:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5f400/0xf74d1861c614afb lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9f2:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2db50 expref: -99 pid: 214199 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346048:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622586cf00. 00010000:00000040:2.0:1775773861.346049:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f620618f200 count: 0 00010000:00000010:2.0:1775773861.346051:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622586c180. 00010000:00000001:2.0:1775773861.346053:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346054:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346055:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233db7600/0xf74d1861c614af4 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9f1:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2db2d expref: -99 pid: 214198 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346059:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6226713780. 00010000:00000040:2.0:1775773861.346060:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0aff00 count: 0 00010000:00000010:2.0:1775773861.346062:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6226713e80. 00010000:00000001:2.0:1775773861.346064:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346065:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346066:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233db6e00/0xf74d1861c614aed lrc: 0/0,0 mode: --/PR res: [0x200002b15:0x9f0:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd88660ed39b2db11 expref: -99 pid: 214198 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00000001:2.0:1775773861.346070:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346071:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346072:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476ca00/0xf74d1861c614ac3 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9ef:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2da70 expref: -99 pid: 214189 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346076:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233b37000. 00010000:00000040:2.0:1775773861.346077:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f621e49d000 count: 0 00010000:00000010:2.0:1775773861.346079:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233b37900. 00010000:00000001:2.0:1775773861.346081:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346082:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346083:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5ec00/0xf74d1861c614abc lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9ee:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2da4d expref: -99 pid: 214188 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346087:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622586c780. 00010000:00000040:2.0:1775773861.346089:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f620618f400 count: 0 00010000:00000010:2.0:1775773861.346090:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622586c680. 00010000:00000001:2.0:1775773861.346093:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346094:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346095:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5f600/0xf74d1861c614ab5 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9ed:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2da2a expref: -99 pid: 214187 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346099:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622586c700. 00010000:00000040:2.0:1775773861.346100:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f620618f800 count: 0 00010000:00000010:2.0:1775773861.346101:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622586c280. 00010000:00000001:2.0:1775773861.346104:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346105:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346106:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5ee00/0xf74d1861c614aae lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9ec:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2da07 expref: -99 pid: 214186 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346110:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622586cd80. 00010000:00000040:2.0:1775773861.346111:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f620618fe00 count: 0 00010000:00000010:2.0:1775773861.346112:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622586cc80. 00010000:00000001:2.0:1775773861.346115:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346115:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346117:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233db6200/0xf74d1861c614aa7 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9eb:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d9e4 expref: -99 pid: 214185 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346120:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6226713880. 00010000:00000040:2.0:1775773861.346121:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0af500 count: 0 00010000:00000010:2.0:1775773861.346123:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6226713e00. 00010000:00000001:2.0:1775773861.346125:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346126:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346127:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5e200/0xf74d1861c614aa0 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9ea:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d9c1 expref: -99 pid: 214184 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346131:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622586c800. 00010000:00000040:2.0:1775773861.346147:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f620618f900 count: 0 00010000:00000010:2.0:1775773861.346149:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622586c200. 00010000:00000001:2.0:1775773861.346152:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346153:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346155:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233db7a00/0xf74d1861c614a99 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9e9:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d99e expref: -99 pid: 214183 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346159:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6226713480. 00010000:00000040:2.0:1775773861.346161:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0afb00 count: 0 00010000:00000010:2.0:1775773861.346162:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6226713a00. 00010000:00000001:2.0:1775773861.346164:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346165:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346166:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5fa00/0xf74d1861c614a92 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9e8:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d97b expref: -99 pid: 214182 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346170:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622586c000. 00010000:00000040:2.0:1775773861.346172:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f620618f700 count: 0 00010000:00000010:2.0:1775773861.346173:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622586c900. 00010000:00000001:2.0:1775773861.346176:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346177:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346179:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e298400/0xf74d1861c614a8b lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9e7:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d958 expref: -99 pid: 214181 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346182:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6202fc4f00. 00010000:00000040:2.0:1775773861.346184:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0afc00 count: 0 00010000:00000010:2.0:1775773861.346185:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6202fc4200. 00010000:00000001:2.0:1775773861.346188:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346189:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346190:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e298800/0xf74d1861c614a84 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9e6:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d935 expref: -99 pid: 214180 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346193:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6202fc4680. 00010000:00000040:2.0:1775773861.346195:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0af300 count: 0 00010000:00000010:2.0:1775773861.346197:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6202fc4c00. 00010000:00000001:2.0:1775773861.346200:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346200:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346202:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5f000/0xf74d1861c614a7d lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9e5:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d912 expref: -99 pid: 214179 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346205:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622586ce80. 00010000:00000040:2.0:1775773861.346207:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f620618fa00 count: 0 00010000:00000010:2.0:1775773861.346208:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622586ca80. 00010000:00000001:2.0:1775773861.346211:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346212:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346213:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e299800/0xf74d1861c614a76 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9e4:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d8ef expref: -99 pid: 214178 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346217:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6202fc4e80. 00010000:00000040:2.0:1775773861.346218:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0af200 count: 0 00010000:00000010:2.0:1775773861.346219:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6202fc4780. 00010000:00000001:2.0:1775773861.346222:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346223:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346224:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5fe00/0xf74d1861c614a6f lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9e3:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d8cc expref: -99 pid: 214177 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346227:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622586c380. 00010000:00000040:2.0:1775773861.346229:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0add00 count: 0 00010000:00000010:2.0:1775773861.346230:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622586cc00. 00010000:00000001:2.0:1775773861.346232:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346233:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346234:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476cc00/0xf74d1861c614a68 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9e2:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d8a9 expref: -99 pid: 214176 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346238:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622e182980. 00010000:00000040:2.0:1775773861.346240:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f621e49d500 count: 0 00010000:00000010:2.0:1775773861.346241:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233b37780. 00010000:00000001:2.0:1775773861.346244:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346245:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346246:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5f200/0xf74d1861c614a61 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9e1:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d886 expref: -99 pid: 214175 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346250:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622586ce00. 00010000:00000040:2.0:1775773861.346251:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0ad400 count: 0 00010000:00000010:2.0:1775773861.346253:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622586cb80. 00010000:00000001:2.0:1775773861.346255:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346256:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346257:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5e400/0xf74d1861c614a5a lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9e0:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d863 expref: -99 pid: 214174 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346261:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622586c480. 00010000:00000040:2.0:1775773861.346262:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0ad800 count: 0 00010000:00000010:2.0:1775773861.346263:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622586ca00. 00010000:00000001:2.0:1775773861.346266:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346266:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346268:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5e800/0xf74d1861c614a53 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9df:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d840 expref: -99 pid: 214173 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346271:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622586c500. 00010000:00000040:2.0:1775773861.346272:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0ade00 count: 0 00010000:00000010:2.0:1775773861.346274:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622586c400. 00010000:00000001:2.0:1775773861.346276:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346277:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346278:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5f800/0xf74d1861c614a4c lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9de:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d81d expref: -99 pid: 214172 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346282:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622586cf80. 00010000:00000040:2.0:1775773861.346283:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0ad900 count: 0 00010000:00000010:2.0:1775773861.346285:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622586c600. 00010000:00000001:2.0:1775773861.346288:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346288:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346290:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5e600/0xf74d1861c614a45 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9dd:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d7fa expref: -99 pid: 214171 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346294:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622586c080. 00010000:00000040:2.0:1775773861.346295:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0ad700 count: 0 00010000:00000010:2.0:1775773861.346296:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622586c580. 00010000:00000001:2.0:1775773861.346299:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346299:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346301:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6208b5e000/0xf74d1861c614a3e lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9dc:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d7d7 expref: -99 pid: 214170 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346304:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622586c980. 00010000:00000040:2.0:1775773861.346305:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0ada00 count: 0 00010000:00000010:2.0:1775773861.346307:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622586c100. 00010000:00000001:2.0:1775773861.346309:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346310:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346311:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476d600/0xf74d1861c614a37 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9db:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d7b4 expref: -99 pid: 214169 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346315:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233b37a80. 00010000:00000040:2.0:1775773861.346316:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f621e49db00 count: 0 00010000:00000010:2.0:1775773861.346317:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233b37380. 00010000:00000001:2.0:1775773861.346320:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346320:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346322:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f55200/0xf74d1861c614a30 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9da:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d791 expref: -99 pid: 214168 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346325:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622e182f00. 00010000:00000040:2.0:1775773861.346327:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622ce91e00 count: 0 00010000:00000010:2.0:1775773861.346328:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622e182400. 00010000:00000001:2.0:1775773861.346331:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346332:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346333:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476d400/0xf74d1861c614a29 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9d9:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d76e expref: -99 pid: 214167 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346337:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233b37c00. 00010000:00000040:2.0:1775773861.346338:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f621e49dc00 count: 0 00010000:00000010:2.0:1775773861.346340:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233b37b00. 00010000:00000001:2.0:1775773861.346342:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346343:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346344:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476c800/0xf74d1861c614a22 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9d8:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d74b expref: -99 pid: 214166 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346347:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233b37880. 00010000:00000040:2.0:1775773861.346349:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6234adcf00 count: 0 00010000:00000010:2.0:1775773861.346350:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233b37e00. 00010000:00000001:2.0:1775773861.346353:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346353:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346355:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f54400/0xf74d1861c614a1b lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9d7:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d728 expref: -99 pid: 214165 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346358:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622586c300. 00010000:00000040:2.0:1775773861.346359:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0ad100 count: 0 00010000:00000010:2.0:1775773861.346361:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621df9d200. 00010000:00000001:2.0:1775773861.346364:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346364:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346366:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f54800/0xf74d1861c614a14 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9d6:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d705 expref: -99 pid: 214164 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346369:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622e182a80. 00010000:00000040:2.0:1775773861.346371:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622ce91900 count: 0 00010000:00000010:2.0:1775773861.346372:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622e182e80. 00010000:00000001:2.0:1775773861.346375:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346376:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346377:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f55800/0xf74d1861c614a0d lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9d5:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d6e2 expref: -99 pid: 214163 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346381:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621df9dc00. 00010000:00000040:2.0:1775773861.346382:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0ad600 count: 0 00010000:00000010:2.0:1775773861.346383:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621df9de80. 00010000:00000001:2.0:1775773861.346386:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346386:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346388:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e298600/0xf74d1861c614a06 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9d4:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d6bf expref: -99 pid: 214162 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346391:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6202fc4700. 00010000:00000040:2.0:1775773861.346392:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0afd00 count: 0 00010000:00000010:2.0:1775773861.346394:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6202fc4180. 00010000:00000001:2.0:1775773861.346396:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346397:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346399:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f54600/0xf74d1861c6149ff lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9d3:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d69c expref: -99 pid: 214161 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346402:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621df9d800. 00010000:00000040:2.0:1775773861.346403:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0adf00 count: 0 00010000:00000010:2.0:1775773861.346405:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621df9dd00. 00010000:00000001:2.0:1775773861.346407:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346408:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346409:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e298000/0xf74d1861c6149f8 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9d2:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d679 expref: -99 pid: 214160 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346412:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6202fc4f80. 00010000:00000040:2.0:1775773861.346414:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0af400 count: 0 00010000:00000010:2.0:1775773861.346415:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6202fc4600. 00010000:00000001:2.0:1775773861.346418:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346419:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346421:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f54000/0xf74d1861c6149f1 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9d1:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d656 expref: -99 pid: 214159 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346424:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621df9de00. 00010000:00000040:2.0:1775773861.346425:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0ad000 count: 0 00010000:00000010:2.0:1775773861.346427:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621df9db80. 00010000:00000001:2.0:1775773861.346429:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346430:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346431:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f54a00/0xf74d1861c6149ea lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9d0:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d633 expref: -99 pid: 214158 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346434:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622e182800. 00010000:00000040:2.0:1775773861.346436:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622ce91700 count: 0 00010000:00000010:2.0:1775773861.346437:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622e182280. 00010000:00000001:2.0:1775773861.346440:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346440:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346442:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f55400/0xf74d1861c6149e3 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9cf:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d610 expref: -99 pid: 214157 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346445:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621df9d480. 00010000:00000040:2.0:1775773861.346446:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0ad500 count: 0 00010000:00000010:2.0:1775773861.346448:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621df9da00. 00010000:00000001:2.0:1775773861.346450:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346451:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346452:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e298a00/0xf74d1861c6149dc lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9ce:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d5ed expref: -99 pid: 214156 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346455:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6202fc4080. 00010000:00000040:2.0:1775773861.346457:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0af800 count: 0 00010000:00000010:2.0:1775773861.346458:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6202fc4580. 00010000:00000001:2.0:1775773861.346461:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346462:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346464:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622d8f7000/0xf74d1861c6149d5 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9cd:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d5ca expref: -99 pid: 214154 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346467:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621df9d500. 00010000:00000040:2.0:1775773861.346480:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0adb00 count: 0 00010000:00000010:2.0:1775773861.346482:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621df9d400. 00010000:00000001:2.0:1775773861.346486:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346486:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346488:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476ce00/0xf74d1861c6149ce lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9cc:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d5a7 expref: -99 pid: 214153 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346492:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233b37480. 00010000:00000040:2.0:1775773861.346493:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6234adc000 count: 0 00010000:00000010:2.0:1775773861.346495:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233b37a00. 00010000:00000001:2.0:1775773861.346498:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346499:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346500:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f55c00/0xf74d1861c6149c7 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9cb:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d584 expref: -99 pid: 214152 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346504:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621df9df80. 00010000:00000040:2.0:1775773861.346505:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0adc00 count: 0 00010000:00000010:2.0:1775773861.346506:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621df9d600. 00010000:00000001:2.0:1775773861.346509:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346509:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346510:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f54c00/0xf74d1861c6149c0 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9ca:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d561 expref: -99 pid: 214151 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346514:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621df9df00. 00010000:00000040:2.0:1775773861.346516:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0ad300 count: 0 00010000:00000010:2.0:1775773861.346517:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621df9d080. 00010000:00000001:2.0:1775773861.346520:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346521:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346522:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622d8f6400/0xf74d1861c6149b9 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9c9:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d53e expref: -99 pid: 214150 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346526:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621df9d180. 00010000:00000040:2.0:1775773861.346527:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0ad200 count: 0 00010000:00000010:2.0:1775773861.346529:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621df9d980. 00010000:00000001:2.0:1775773861.346531:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346532:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346533:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f55600/0xf74d1861c6149b2 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9c8:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d51b expref: -99 pid: 214149 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346537:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6202fc4980. 00010000:00000040:2.0:1775773861.346538:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0afe00 count: 0 00010000:00000010:2.0:1775773861.346540:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6202fc4100. 00010000:00000001:2.0:1775773861.346542:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346543:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346544:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476d200/0xf74d1861c6149ab lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9c7:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d4f8 expref: -99 pid: 214148 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346548:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233b37400. 00010000:00000040:2.0:1775773861.346549:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6234adc500 count: 0 00010000:00000010:2.0:1775773861.346550:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233b37f80. 00010000:00000001:2.0:1775773861.346553:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346554:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346555:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f54e00/0xf74d1861c6149a4 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9c6:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d4d5 expref: -99 pid: 214147 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346559:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621df9d100. 00010000:00000040:2.0:1775773861.346560:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233961d00 count: 0 00010000:00000010:2.0:1775773861.346562:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621df9d300. 00010000:00000001:2.0:1775773861.346565:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346566:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346567:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476c200/0xf74d1861c61499d lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9c5:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d4b2 expref: -99 pid: 214146 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346571:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621df9d280. 00010000:00000040:2.0:1775773861.346572:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233961400 count: 0 00010000:00000010:2.0:1775773861.346574:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621df9d700. 00010000:00000001:2.0:1775773861.346576:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346577:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346578:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f54200/0xf74d1861c614996 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9c4:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d48f expref: -99 pid: 214145 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346582:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621df9dd80. 00010000:00000040:2.0:1775773861.346583:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233961800 count: 0 00010000:00000010:2.0:1775773861.346584:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621df9dc80. 00010000:00000001:2.0:1775773861.346586:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346587:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346588:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e299400/0xf74d1861c61498f lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9c3:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d46c expref: -99 pid: 214144 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346592:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6202fc4300. 00010000:00000040:2.0:1775773861.346593:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622e0af900 count: 0 00010000:00000010:2.0:1775773861.346595:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6202fc4280. 00010000:00000001:2.0:1775773861.346597:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346598:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346599:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622d8f7a00/0xf74d1861c614988 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9c2:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d449 expref: -99 pid: 214143 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346603:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622e182700. 00010000:00000040:2.0:1775773861.346604:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622ce91a00 count: 0 00010000:00000010:2.0:1775773861.346606:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622e182d80. 00010000:00000001:2.0:1775773861.346609:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346609:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346611:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622d8f7600/0xf74d1861c614981 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9c1:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d426 expref: -99 pid: 214142 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346615:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233b37080. 00010000:00000040:2.0:1775773861.346616:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6234adcb00 count: 0 00010000:00000010:2.0:1775773861.346617:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233b37580. 00010000:00000001:2.0:1775773861.346620:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346621:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346622:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622d8f6c00/0xf74d1861c61497a lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9c0:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d403 expref: -99 pid: 214141 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346625:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6202fc4d80. 00010000:00000040:2.0:1775773861.346627:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6202d82f00 count: 0 00010000:00000010:2.0:1775773861.346628:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6202fc4c80. 00010000:00000001:2.0:1775773861.346630:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346631:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346632:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e299c00/0xf74d1861c614973 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9bf:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d3e0 expref: -99 pid: 214140 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346636:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6202fc4d00. 00010000:00000040:2.0:1775773861.346637:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6202d82000 count: 0 00010000:00000010:2.0:1775773861.346638:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6202fc4800. 00010000:00000001:2.0:1775773861.346641:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346641:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346643:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f55a00/0xf74d1861c61496c lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9be:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d3bd expref: -99 pid: 214139 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346646:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621df9d680. 00010000:00000040:2.0:1775773861.346648:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233961e00 count: 0 00010000:00000010:2.0:1775773861.346649:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621df9d000. 00010000:00000001:2.0:1775773861.346652:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346653:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346654:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f55000/0xf74d1861c614965 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9bd:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d39a expref: -99 pid: 214138 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346658:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621df9d900. 00010000:00000040:2.0:1775773861.346659:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233961900 count: 0 00010000:00000010:2.0:1775773861.346660:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621df9d780. 00010000:00000001:2.0:1775773861.346663:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346663:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346665:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6233f55e00/0xf74d1861c61495e lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9bc:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d377 expref: -99 pid: 214137 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346668:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621df9da80. 00010000:00000040:2.0:1775773861.346669:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233961700 count: 0 00010000:00000010:2.0:1775773861.346671:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621df9d380. 00010000:00000001:2.0:1775773861.346673:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346674:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346675:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622d8f6a00/0xf74d1861c614957 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9bb:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d354 expref: -99 pid: 214136 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346678:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6202fc4000. 00010000:00000040:2.0:1775773861.346680:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6202d82500 count: 0 00010000:00000010:2.0:1775773861.346681:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6202fc4900. 00010000:00000001:2.0:1775773861.346683:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346684:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346685:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c0400/0xf74d1861c614950 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9ba:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d331 expref: -99 pid: 214135 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346689:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622e182c80. 00010000:00000040:2.0:1775773861.346690:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622ce91100 count: 0 00010000:00000010:2.0:1775773861.346692:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622e182d00. 00010000:00000001:2.0:1775773861.346695:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346696:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346697:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e298c00/0xf74d1861c614949 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9b9:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d30e expref: -99 pid: 214134 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346701:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6202fc4a80. 00010000:00000040:2.0:1775773861.346702:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6202d82b00 count: 0 00010000:00000010:2.0:1775773861.346703:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6202fc4380. 00010000:00000001:2.0:1775773861.346705:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346706:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346707:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c0800/0xf74d1861c614942 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9b8:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d2eb expref: -99 pid: 214133 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346711:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621df9db00. 00010000:00000040:2.0:1775773861.346712:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233961a00 count: 0 00010000:00000010:2.0:1775773861.346713:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621df9d880. 00010000:00000001:2.0:1775773861.346716:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346716:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346718:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e299600/0xf74d1861c61493b lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9b7:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d2c8 expref: -99 pid: 214132 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346721:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6202fc4b00. 00010000:00000040:2.0:1775773861.346722:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6202d82c00 count: 0 00010000:00000010:2.0:1775773861.346724:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6202fc4880. 00010000:00000001:2.0:1775773861.346726:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346727:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346728:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c1800/0xf74d1861c614934 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9b6:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d2a5 expref: -99 pid: 214131 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346732:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233b37300. 00010000:00000040:2.0:1775773861.346733:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6234adcc00 count: 0 00010000:00000010:2.0:1775773861.346735:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233b37280. 00010000:00000001:2.0:1775773861.346738:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346739:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346740:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62265d7000/0xf74d1861c61492d lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9b5:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d282 expref: -99 pid: 214130 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346744:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233b37700. 00010000:00000040:2.0:1775773861.346745:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6234adc300 count: 0 00010000:00000010:2.0:1775773861.346746:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233b37d80. 00010000:00000001:2.0:1775773861.346749:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346750:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346751:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c0600/0xf74d1861c614926 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9b4:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d25f expref: -99 pid: 214129 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346754:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233640f00. 00010000:00000040:2.0:1775773861.346756:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233961100 count: 0 00010000:00000010:2.0:1775773861.346757:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233640680. 00010000:00000001:2.0:1775773861.346760:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346761:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346762:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c0000/0xf74d1861c61491f lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9b3:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d23c expref: -99 pid: 214128 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346765:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233640a80. 00010000:00000040:2.0:1775773861.346767:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233961600 count: 0 00010000:00000010:2.0:1775773861.346768:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233640b00. 00010000:00000001:2.0:1775773861.346771:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346771:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346773:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e298e00/0xf74d1861c614918 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9b2:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d219 expref: -99 pid: 214127 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346777:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6202fc4e00. 00010000:00000040:2.0:1775773861.346778:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6202d82300 count: 0 00010000:00000010:2.0:1775773861.346780:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6202fc4b80. 00010000:00000001:2.0:1775773861.346783:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346784:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346785:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e298200/0xf74d1861c614911 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9b1:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d1f6 expref: -99 pid: 214126 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346789:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6202fc4a00. 00010000:00000040:2.0:1775773861.346790:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6202d82200 count: 0 00010000:00000010:2.0:1775773861.346791:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6202fc4500. 00010000:00000001:2.0:1775773861.346794:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346794:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346796:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c0a00/0xf74d1861c61490a lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9b0:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d1d3 expref: -99 pid: 214125 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346799:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6202fc4400. 00010000:00000040:2.0:1775773861.346800:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6202d82d00 count: 0 00010000:00000010:2.0:1775773861.346802:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621dcae380. 00010000:00000001:2.0:1775773861.346806:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346807:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346808:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c1400/0xf74d1861c614903 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9af:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d1b0 expref: -99 pid: 214124 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346812:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233640700. 00010000:00000040:2.0:1775773861.346813:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233961f00 count: 0 00010000:00000010:2.0:1775773861.346814:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233640080. 00010000:00000001:2.0:1775773861.346817:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346818:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346819:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e299a00/0xf74d1861c6148fc lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9ae:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d18d expref: -99 pid: 214123 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346823:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621dcae580. 00010000:00000040:2.0:1775773861.346824:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6202d82400 count: 0 00010000:00000010:2.0:1775773861.346826:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621dcae400. 00010000:00000001:2.0:1775773861.346829:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346830:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346831:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e299000/0xf74d1861c6148f5 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9ad:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d16a expref: -99 pid: 214122 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346835:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621dcae280. 00010000:00000040:2.0:1775773861.346836:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6202d82800 count: 0 00010000:00000010:2.0:1775773861.346837:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621dcae800. 00010000:00000001:2.0:1775773861.346840:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346840:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346842:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c1c00/0xf74d1861c6148ee lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9ac:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d147 expref: -99 pid: 214121 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346845:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621dcaeb00. 00010000:00000040:2.0:1775773861.346846:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6202d82e00 count: 0 00010000:00000010:2.0:1775773861.346848:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621dcaec00. 00010000:00000001:2.0:1775773861.346850:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346851:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346852:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c0c00/0xf74d1861c6148e7 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9ab:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d124 expref: -99 pid: 214120 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346856:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233640f80. 00010000:00000040:2.0:1775773861.346857:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233961000 count: 0 00010000:00000010:2.0:1775773861.346858:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233640600. 00010000:00000001:2.0:1775773861.346861:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346861:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346863:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62265d6e00/0xf74d1861c6148e0 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9aa:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d101 expref: -99 pid: 214119 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346866:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233b37c80. 00010000:00000040:2.0:1775773861.346868:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6234adc200 count: 0 00010000:00000010:2.0:1775773861.346869:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233b37d00. 00010000:00000001:2.0:1775773861.346872:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346873:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346874:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c1600/0xf74d1861c6148d9 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9a9:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d0de expref: -99 pid: 214118 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346878:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622cc49500. 00010000:00000040:2.0:1775773861.346879:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6234adcd00 count: 0 00010000:00000010:2.0:1775773861.346881:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622cc49e80. 00010000:00000001:2.0:1775773861.346883:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346884:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346885:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c0e00/0xf74d1861c6148d2 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9a8:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d0bb expref: -99 pid: 214117 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346888:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621dcae780. 00010000:00000040:2.0:1775773861.346890:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6202d82900 count: 0 00010000:00000010:2.0:1775773861.346891:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621dcae200. 00010000:00000001:2.0:1775773861.346894:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346895:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346896:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476c600/0xf74d1861c6148cb lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9a7:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d098 expref: -99 pid: 214116 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346900:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622cc49300. 00010000:00000040:2.0:1775773861.346901:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6234adc400 count: 0 00010000:00000010:2.0:1775773861.346902:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622cc49d80. 00010000:00000001:2.0:1775773861.346905:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346905:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346907:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c0200/0xf74d1861c6148c4 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9a6:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d075 expref: -99 pid: 214115 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346910:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233640580. 00010000:00000040:2.0:1775773861.346912:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233961500 count: 0 00010000:00000010:2.0:1775773861.346913:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233640180. 00010000:00000001:2.0:1775773861.346916:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346917:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346918:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e299e00/0xf74d1861c6148bd lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9a5:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d052 expref: -99 pid: 214114 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346922:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621dcaed80. 00010000:00000040:2.0:1775773861.346923:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6202d82700 count: 0 00010000:00000010:2.0:1775773861.346925:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621dcae700. 00010000:00000001:2.0:1775773861.346927:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346928:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346929:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62265d6400/0xf74d1861c6148b6 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9a4:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d02f expref: -99 pid: 214113 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346933:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622e182200. 00010000:00000040:2.0:1775773861.346934:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622ce91600 count: 0 00010000:00000010:2.0:1775773861.346935:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622e182680. 00010000:00000001:2.0:1775773861.346938:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346938:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346940:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476da00/0xf74d1861c6148af lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9a3:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2d00c expref: -99 pid: 214112 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346943:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622e182900. 00010000:00000040:2.0:1775773861.346945:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622ce91f00 count: 0 00010000:00000010:2.0:1775773861.346946:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622e182780. 00010000:00000001:2.0:1775773861.346948:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346949:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346950:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476de00/0xf74d1861c6148a8 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9a2:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cfe9 expref: -99 pid: 214111 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346954:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233640980. 00010000:00000040:2.0:1775773861.346955:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233961b00 count: 0 00010000:00000010:2.0:1775773861.346957:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233640100. 00010000:00000001:2.0:1775773861.346960:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346961:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346962:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476c400/0xf74d1861c6148a1 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9a1:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cfc6 expref: -99 pid: 214110 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346966:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233640300. 00010000:00000040:2.0:1775773861.346967:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233961c00 count: 0 00010000:00000010:2.0:1775773861.346968:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233640280. 00010000:00000001:2.0:1775773861.346971:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346971:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346973:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622e299200/0xf74d1861c61489a lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x9a0:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cfa3 expref: -99 pid: 214109 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346976:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621dcae300. 00010000:00000040:2.0:1775773861.346977:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6202d82a00 count: 0 00010000:00000010:2.0:1775773861.346979:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621dcae180. 00010000:00000001:2.0:1775773861.346981:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346982:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346983:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c1a00/0xf74d1861c614893 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x99f:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cf80 expref: -99 pid: 214108 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346987:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233640c80. 00010000:00000040:2.0:1775773861.346988:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233961300 count: 0 00010000:00000010:2.0:1775773861.346989:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233640d00. 00010000:00000001:2.0:1775773861.346992:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.346992:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.346993:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c1000/0xf74d1861c61488c lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x99e:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cf5d expref: -99 pid: 214107 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.346997:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233640800. 00010000:00000040:2.0:1775773861.346999:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233961200 count: 0 00010000:00000010:2.0:1775773861.347001:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233640200. 00010000:00000001:2.0:1775773861.347004:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347004:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347006:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c1e00/0xf74d1861c614885 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x99d:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cf3a expref: -99 pid: 214106 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347009:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622e182380. 00010000:00000040:2.0:1775773861.347011:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622ce91000 count: 0 00010000:00000010:2.0:1775773861.347012:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622e182c00. 00010000:00000001:2.0:1775773861.347015:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347015:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347017:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62267c1200/0xf74d1861c61487e lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x99c:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cf17 expref: -99 pid: 214105 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347020:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233640000. 00010000:00000040:2.0:1775773861.347021:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233960b00 count: 0 00010000:00000010:2.0:1775773861.347022:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233640900. 00010000:00000001:2.0:1775773861.347025:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347026:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347027:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226128e00/0xf74d1861c614877 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x99b:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cef4 expref: -99 pid: 214104 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347031:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233640780. 00010000:00000040:2.0:1775773861.347032:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233960c00 count: 0 00010000:00000010:2.0:1775773861.347033:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233640e80. 00010000:00000001:2.0:1775773861.347036:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347037:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347038:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f622c1ec200/0xf74d1861c614870 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x99a:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2ced1 expref: -99 pid: 214103 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347042:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622e182b00. 00010000:00000040:2.0:1775773861.347043:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622ce91500 count: 0 00010000:00000010:2.0:1775773861.347045:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622e182880. 00010000:00000001:2.0:1775773861.347048:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347049:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347050:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226128200/0xf74d1861c614869 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x999:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2ceae expref: -99 pid: 214102 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347054:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233640380. 00010000:00000040:2.0:1775773861.347055:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233960300 count: 0 00010000:00000010:2.0:1775773861.347056:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233640c00. 00010000:00000001:2.0:1775773861.347059:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347059:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347061:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226129a00/0xf74d1861c614862 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x998:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2ce8b expref: -99 pid: 214101 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347064:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233640880. 00010000:00000040:2.0:1775773861.347065:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233960200 count: 0 00010000:00000010:2.0:1775773861.347067:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233640e00. 00010000:00000001:2.0:1775773861.347069:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347070:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347071:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226129000/0xf74d1861c61485b lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x997:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2ce68 expref: -99 pid: 214100 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347075:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233640b80. 00010000:00000040:2.0:1775773861.347076:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233960d00 count: 0 00010000:00000010:2.0:1775773861.347077:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233640480. 00010000:00000001:2.0:1775773861.347079:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347080:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347082:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226129e00/0xf74d1861c614854 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x996:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2ce45 expref: -99 pid: 214099 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347085:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622e182b80. 00010000:00000040:2.0:1775773861.347086:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622ce91b00 count: 0 00010000:00000010:2.0:1775773861.347088:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622e182480. 00010000:00000001:2.0:1775773861.347091:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347092:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347093:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f621839e800/0xf74d1861c61484d lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x995:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2ce22 expref: -99 pid: 214098 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347097:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233640a00. 00010000:00000040:2.0:1775773861.347098:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233960400 count: 0 00010000:00000010:2.0:1775773861.347099:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6233640500. 00010000:00000001:2.0:1775773861.347101:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347102:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347103:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62265d6200/0xf74d1861c614846 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x994:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cdff expref: -99 pid: 214097 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347107:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622e182a00. 00010000:00000040:2.0:1775773861.347108:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6206bb7300 count: 0 00010000:00000010:2.0:1775773861.347110:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622e182500. 00010000:00000001:2.0:1775773861.347112:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347113:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347114:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226129200/0xf74d1861c61483f lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x993:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cddc expref: -99 pid: 214096 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347117:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6233640400. 00010000:00000040:2.0:1775773861.347119:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233960800 count: 0 00010000:00000010:2.0:1775773861.347120:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6206adf080. 00010000:00000001:2.0:1775773861.347122:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347123:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347124:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f621839f800/0xf74d1861c614838 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x992:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cdb9 expref: -99 pid: 214095 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347128:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621dcae080. 00010000:00000040:2.0:1775773861.347130:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6202d82100 count: 0 00010000:00000010:2.0:1775773861.347131:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621dcaef80. 00010000:00000001:2.0:1775773861.347148:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347149:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347150:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226128400/0xf74d1861c614831 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x991:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cd96 expref: -99 pid: 214094 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347155:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6206adfd80. 00010000:00000040:2.0:1775773861.347157:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233960e00 count: 0 00010000:00000010:2.0:1775773861.347158:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6206adf300. 00010000:00000001:2.0:1775773861.347161:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347162:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347163:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476dc00/0xf74d1861c61482a lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x990:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cd73 expref: -99 pid: 214093 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347167:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622cc49180. 00010000:00000040:2.0:1775773861.347168:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6234adc800 count: 0 00010000:00000010:2.0:1775773861.347169:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622cc49f00. 00010000:00000001:2.0:1775773861.347172:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347173:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347174:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226128800/0xf74d1861c614823 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x98f:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cd50 expref: -99 pid: 214092 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347178:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6206adff80. 00010000:00000040:2.0:1775773861.347179:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233960900 count: 0 00010000:00000010:2.0:1775773861.347180:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6206adfe00. 00010000:00000001:2.0:1775773861.347183:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347184:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347185:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f62265d7a00/0xf74d1861c61481c lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x98e:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cd2d expref: -99 pid: 214091 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347189:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621dcae480. 00010000:00000040:2.0:1775773861.347191:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6202d82600 count: 0 00010000:00000010:2.0:1775773861.347192:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621dcaeb80. 00010000:00000001:2.0:1775773861.347195:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347196:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347197:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226129800/0xf74d1861c614815 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x98d:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cd0a expref: -99 pid: 214090 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347201:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6206adfc00. 00010000:00000040:2.0:1775773861.347202:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233960700 count: 0 00010000:00000010:2.0:1775773861.347203:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6206adf900. 00010000:00000001:2.0:1775773861.347206:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347207:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347208:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f623476d000/0xf74d1861c61480e lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x98c:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cce7 expref: -99 pid: 214089 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347212:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f622e182f80. 00010000:00000040:2.0:1775773861.347213:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6206bb7a00 count: 0 00010000:00000010:2.0:1775773861.347214:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f622e182600. 00010000:00000001:2.0:1775773861.347217:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347217:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347219:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226128600/0xf74d1861c614807 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x98b:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2ccc4 expref: -99 pid: 214088 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347222:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621dcaea00. 00010000:00000040:2.0:1775773861.347223:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622d964200 count: 0 00010000:00000010:2.0:1775773861.347225:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621dcae500. 00010000:00000001:2.0:1775773861.347227:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347228:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347229:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f621839e600/0xf74d1861c614800 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x98a:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cca1 expref: -99 pid: 214087 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347233:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f621dcae600. 00010000:00000040:2.0:1775773861.347234:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f622d964d00 count: 0 00010000:00000010:2.0:1775773861.347236:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f621dcaef00. 00010000:00000001:2.0:1775773861.347239:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347239:0:207610:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1775773861.347241:0:207610:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f62106eb800 lock: ffff8f6226128000/0xf74d1861c6147f9 lrc: 0/0,0 mode: --/PW res: [0x200002b15:0x989:0x0].0x0 bits 0x5b/0x0 rrc: 2 type: IBT gid 0 flags: 0x804829400000020 nid: local remote: 0xd88660ed39b2cc7e expref: -99 pid: 214086 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00000010:2.0:1775773861.347244:0:207610:0:(ldlm_lock.c:209:ldlm_lock_put()) kvfree 'lock->l_lvb_data': 112 at ffff8f6206adf780. 00010000:00000040:2.0:1775773861.347246:0:207610:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff8f6233960a00 count: 0 00010000:00000010:2.0:1775773861.347247:0:207610:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff8f6206adfe80. 00010000:00000001:2.0:1775773861.347249:0:207610:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1775773861.347250:0:207610:0:(ldlm_request.c:2492:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.0:1775773861.347252:0:207610:0:(ldlm_lockd.c:2913:ldlm_bl_thread_blwi()) kfreed 'blwi': 192 at ffff8f622e48fe40. 00010000:00000001:2.0:1775773861.347254:0:207610:0:(ldlm_lockd.c:2917:ldlm_bl_thread_blwi()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773861.347257:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773861.347258:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773861.347260:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773861.347261:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773861.347262:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773861.347263:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773861.347264:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773861.347265:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773861.347266:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000010:2.1:1775773861.349182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380860. 00000080:00000010:2.1:1775773861.353166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62263807e0. 00000080:00000010:2.1:1775773861.353174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6234946220. 00000080:00000010:2.1:1775773861.353179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380720. 00000080:00000010:2.1:1775773861.353184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380040. 00000080:00000010:2.1:1775773861.353189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62263803a0. 00000080:00000010:2.1:1775773861.353192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380840. 00000080:00000010:2.1:1775773861.353195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380000. 00000080:00000010:2.1:1775773861.353198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62263807a0. 00000080:00000010:2.1:1775773861.353200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62263809a0. 00000080:00000010:2.1:1775773861.353203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25b20. 00000080:00000010:2.1:1775773861.353207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70820. 00000080:00000010:2.1:1775773861.353211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c256c0. 00000080:00000010:2.1:1775773861.353213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380680. 00000080:00000010:2.1:1775773861.353216:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62263800e0. 00000080:00000010:2.1:1775773861.353219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380120. 00000080:00000010:2.1:1775773861.353221:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62263806c0. 00000080:00000010:2.1:1775773861.353228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380580. 00000080:00000010:2.1:1775773861.353231:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25620. 00000080:00000010:2.1:1775773861.353235:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25400. 00000080:00000010:2.1:1775773861.353238:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25a20. 00000080:00000010:2.1:1775773861.353242:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380f80. 00000080:00000010:2.1:1775773861.353244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6234946900. 00000080:00000010:2.1:1775773861.353247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25be0. 00000080:00000010:2.1:1775773861.353250:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380da0. 00000080:00000010:2.1:1775773861.353252:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380420. 00000080:00000010:2.1:1775773861.353255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380320. 00000080:00000010:2.1:1775773861.353258:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6234946ea0. 00000080:00000010:2.1:1775773861.353262:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380160. 00000080:00000010:2.1:1775773861.353264:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6234946020. 00000080:00000010:2.1:1775773861.353267:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380e20. 00000080:00000010:2.1:1775773861.353272:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6234946120. 00000080:00000010:2.1:1775773861.353275:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62349463e0. 00000080:00000010:2.1:1775773861.353278:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380ae0. 00000080:00000010:2.1:1775773861.353281:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6234946460. 00000080:00000010:2.1:1775773861.353283:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62263803e0. 00000080:00000010:2.1:1775773861.353286:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25ac0. 00000080:00000010:2.1:1775773861.353288:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62263809e0. 00000080:00000010:2.1:1775773861.353291:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62263806a0. 00000080:00000010:2.1:1775773861.353293:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380c40. 00000080:00000010:2.1:1775773861.353296:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380100. 00000080:00000010:2.1:1775773861.353298:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380800. 00000080:00000010:2.1:1775773861.353300:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380ee0. 00000080:00000010:2.1:1775773861.353303:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25560. 00000080:00000010:2.1:1775773861.353305:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380a00. 00000080:00000010:2.1:1775773861.353307:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25d60. 00000080:00000010:2.1:1775773861.353310:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c258a0. 00000080:00000010:2.1:1775773861.353312:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380020. 00000080:00000010:2.1:1775773861.353314:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62263805a0. 00000080:00000010:2.1:1775773861.353316:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380540. 00000080:00000010:2.1:1775773861.353318:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380b00. 00000080:00000010:2.1:1775773861.353321:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380760. 00000080:00000010:2.1:1775773861.353324:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62349468a0. 00000080:00000010:2.1:1775773861.353326:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380d80. 00000080:00000010:2.1:1775773861.353329:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380500. 00000080:00000010:2.1:1775773861.353331:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62263806e0. 00000080:00000010:2.1:1775773861.353334:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6234946100. 00000080:00000010:2.1:1775773861.353336:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70920. 00000080:00000010:2.1:1775773861.353339:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25b00. 00000080:00000010:2.1:1775773861.353341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380d40. 00000080:00000010:2.1:1775773861.353344:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380480. 00000080:00000010:2.1:1775773861.353346:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70c00. 00000080:00000010:2.1:1775773861.353348:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62263801e0. 00000080:00000010:2.1:1775773861.353350:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25bc0. 00000080:00000010:2.1:1775773861.353353:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380ce0. 00000080:00000010:2.1:1775773861.353355:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25140. 00000080:00000010:2.1:1775773861.353358:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380220. 00000080:00000010:2.1:1775773861.353360:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6234946980. 00000080:00000010:2.1:1775773861.353363:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df708e0. 00000080:00000010:2.1:1775773861.353366:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70000. 00000080:00000010:2.1:1775773861.353368:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70800. 00000080:00000010:2.1:1775773861.353371:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6234946f60. 00000080:00000010:2.1:1775773861.353373:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380140. 00000080:00000010:2.1:1775773861.353376:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380b60. 00000080:00000010:2.1:1775773861.353379:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62263804a0. 00000080:00000010:2.1:1775773861.353383:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70d60. 00000080:00000010:2.1:1775773861.353387:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380ca0. 00000080:00000010:2.1:1775773861.353391:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6234946a00. 00000080:00000010:2.1:1775773861.353394:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380920. 00000080:00000010:2.1:1775773861.353398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62349462a0. 00000080:00000010:2.1:1775773861.353401:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62263808a0. 00000080:00000010:2.1:1775773861.353405:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70d00. 00000080:00000010:2.1:1775773861.353409:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6226380f60. 00000080:00000010:2.1:1775773861.353413:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06c0c0. 00000080:00000010:2.1:1775773861.353417:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6234946d00. 00000080:00000010:2.1:1775773861.353421:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62349466e0. 00000080:00000010:2.1:1775773861.353425:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f622e06ca00. 00000080:00000010:2.1:1775773861.353430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225a14fe0. 00000080:00000010:2.1:1775773861.353434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6234946dc0. 00000080:00000010:2.1:1775773861.353438:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6234946d40. 00000080:00000010:2.1:1775773861.353442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6210db4880. 00000080:00000010:2.1:1775773861.353446:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7280. 00000080:00000010:2.1:1775773861.353448:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70640. 00000080:00000010:2.1:1775773861.353451:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7a00. 00000080:00000010:2.1:1775773861.353454:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225a14fc0. 00000080:00000010:2.1:1775773861.353457:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25f60. 00000080:00000010:2.1:1775773861.353459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6225a140e0. 00000080:00000010:2.1:1775773861.353462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6234946240. 00000080:00000010:2.1:1775773861.353464:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70e20. 00000080:00000010:2.1:1775773861.353466:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25060. 00000080:00000010:2.1:1775773861.353481:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25d80. 00000080:00000010:2.1:1775773861.353484:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25160. 00000080:00000010:2.1:1775773861.353486:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62349461a0. 00000080:00000010:2.1:1775773861.353489:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7cc0. 00000080:00000010:2.1:1775773861.353492:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7d80. 00000080:00000010:2.1:1775773861.353494:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7f60. 00000080:00000010:2.1:1775773861.353496:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7920. 00000080:00000010:2.1:1775773861.353498:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7860. 00000080:00000010:2.1:1775773861.353500:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6234946300. 00000080:00000010:2.1:1775773861.353503:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62349467a0. 00000080:00000010:2.1:1775773861.353505:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7080. 00000080:00000010:2.1:1775773861.353507:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7680. 00000080:00000010:2.1:1775773861.353509:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7420. 00000080:00000010:2.1:1775773861.353511:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c73c0. 00000080:00000010:2.1:1775773861.353513:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70060. 00000080:00000010:2.1:1775773861.353515:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7040. 00000080:00000010:2.1:1775773861.353517:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6206644e00. 00000080:00000010:2.1:1775773861.353520:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7e60. 00000080:00000010:2.1:1775773861.353523:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25d20. 00000080:00000010:2.1:1775773861.353524:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7f40. 00000080:00000010:2.1:1775773861.353526:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f621df70c40. 00000080:00000010:2.1:1775773861.353528:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c74a0. 00000080:00000010:2.1:1775773861.353530:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6207c25e40. 00000080:00000010:2.1:1775773861.353532:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7c60. 00000080:00000010:2.1:1775773861.353534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f6206644c60. 00000080:00000010:2.1:1775773861.353536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c73e0. 00000080:00000010:2.1:1775773861.353538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff8f62063c7100. 00010000:00000010:2.1:1775773861.353541:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b66d00. 00010000:00000010:2.1:1775773861.353545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c0200. 00010000:00000010:2.1:1775773861.353547:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b66400. 00010000:00000010:2.1:1775773861.353549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c1a00. 00010000:00000010:2.1:1775773861.353551:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0afa00. 00010000:00000010:2.1:1775773861.353553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233db6c00. 00010000:00000010:2.1:1775773861.353555:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b66800. 00010000:00000010:2.1:1775773861.353557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c1000. 00010000:00000010:2.1:1775773861.353559:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b66e00. 00010000:00000010:2.1:1775773861.353561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c1e00. 00010000:00000010:2.1:1775773861.353563:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206b66900. 00010000:00000010:2.1:1775773861.353564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c1200. 00010000:00000010:2.1:1775773861.353567:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618fd00. 00010000:00000010:2.1:1775773861.353569:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c0400. 00010000:00000010:2.1:1775773861.353571:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618f100. 00010000:00000010:2.1:1775773861.353573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c0800. 00010000:00000010:2.1:1775773861.353574:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618f600. 00010000:00000010:2.1:1775773861.353576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c1800. 00010000:00000010:2.1:1775773861.353578:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618ff00. 00010000:00000010:2.1:1775773861.353579:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c0600. 00010000:00000010:2.1:1775773861.353581:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618f000. 00010000:00000010:2.1:1775773861.353583:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc0d200. 00010000:00000010:2.1:1775773861.353585:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce91400. 00010000:00000010:2.1:1775773861.353587:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8f6800. 00010000:00000010:2.1:1775773861.353588:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce91800. 00010000:00000010:2.1:1775773861.353590:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc0d600. 00010000:00000010:2.1:1775773861.353591:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618f500. 00010000:00000010:2.1:1775773861.353593:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c0000. 00010000:00000010:2.1:1775773861.353595:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0af100. 00010000:00000010:2.1:1775773861.353597:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c0a00. 00010000:00000010:2.1:1775773861.353598:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618fb00. 00010000:00000010:2.1:1775773861.353600:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62264c1400. 00010000:00000010:2.1:1775773861.353602:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618fc00. 00010000:00000010:2.1:1775773861.353604:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b5fc00. 00010000:00000010:2.1:1775773861.353606:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0af600. 00010000:00000010:2.1:1775773861.353608:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b5ea00. 00010000:00000010:2.1:1775773861.353609:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618f300. 00010000:00000010:2.1:1775773861.353611:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc0da00. 00010000:00000010:2.1:1775773861.353618:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e49d600. 00010000:00000010:2.1:1775773861.353620:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622cc0d000. 00010000:00000010:2.1:1775773861.353622:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e49df00. 00010000:00000010:2.1:1775773861.353624:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623476d800. 00010000:00000010:2.1:1775773861.353626:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618f200. 00010000:00000010:2.1:1775773861.353628:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b5f400. 00010000:00000010:2.1:1775773861.353630:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0aff00. 00010000:00000010:2.1:1775773861.353632:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233db7600. 00010000:00000010:2.1:1775773861.353633:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233db6e00. 00010000:00000010:2.1:1775773861.353635:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e49d000. 00010000:00000010:2.1:1775773861.353637:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623476ca00. 00010000:00000010:2.1:1775773861.353638:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618f400. 00010000:00000010:2.1:1775773861.353640:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b5ec00. 00010000:00000010:2.1:1775773861.353641:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618f800. 00010000:00000010:2.1:1775773861.353643:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b5f600. 00010000:00000010:2.1:1775773861.353645:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618fe00. 00010000:00000010:2.1:1775773861.353646:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b5ee00. 00010000:00000010:2.1:1775773861.353648:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0af500. 00010000:00000010:2.1:1775773861.353649:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233db6200. 00010000:00000010:2.1:1775773861.353651:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618f900. 00010000:00000010:2.1:1775773861.353652:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b5e200. 00010000:00000010:2.1:1775773861.353654:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0afb00. 00010000:00000010:2.1:1775773861.353655:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233db7a00. 00010000:00000010:2.1:1775773861.353657:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618f700. 00010000:00000010:2.1:1775773861.353658:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b5fa00. 00010000:00000010:2.1:1775773861.353660:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0afc00. 00010000:00000010:2.1:1775773861.353662:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e298400. 00010000:00000010:2.1:1775773861.353664:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0af300. 00010000:00000010:2.1:1775773861.353665:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e298800. 00010000:00000010:2.1:1775773861.353667:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f620618fa00. 00010000:00000010:2.1:1775773861.353668:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b5f000. 00010000:00000010:2.1:1775773861.353670:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0af200. 00010000:00000010:2.1:1775773861.353672:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e299800. 00010000:00000010:2.1:1775773861.353674:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0add00. 00010000:00000010:2.1:1775773861.353675:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b5fe00. 00010000:00000010:2.1:1775773861.353677:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e49d500. 00010000:00000010:2.1:1775773861.353679:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623476cc00. 00010000:00000010:2.1:1775773861.353681:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0ad400. 00010000:00000010:2.1:1775773861.353683:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b5f200. 00010000:00000010:2.1:1775773861.353684:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0ad800. 00010000:00000010:2.1:1775773861.353686:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b5e400. 00010000:00000010:2.1:1775773861.353688:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0ade00. 00010000:00000010:2.1:1775773861.353690:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b5e800. 00010000:00000010:2.1:1775773861.353691:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0ad900. 00010000:00000010:2.1:1775773861.353693:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b5f800. 00010000:00000010:2.1:1775773861.353694:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0ad700. 00010000:00000010:2.1:1775773861.353696:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b5e600. 00010000:00000010:2.1:1775773861.353697:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0ada00. 00010000:00000010:2.1:1775773861.353699:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6208b5e000. 00010000:00000010:2.1:1775773861.353700:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e49db00. 00010000:00000010:2.1:1775773861.353702:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623476d600. 00010000:00000010:2.1:1775773861.353707:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce91e00. 00010000:00000010:2.1:1775773861.353709:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233f55200. 00010000:00000010:2.1:1775773861.353711:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f621e49dc00. 00010000:00000010:2.1:1775773861.353712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623476d400. 00010000:00000010:2.1:1775773861.353714:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234adcf00. 00010000:00000010:2.1:1775773861.353716:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623476c800. 00010000:00000010:2.1:1775773861.353717:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0ad100. 00010000:00000010:2.1:1775773861.353719:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233f54400. 00010000:00000010:2.1:1775773861.353721:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce91900. 00010000:00000010:2.1:1775773861.353722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233f54800. 00010000:00000010:2.1:1775773861.353724:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0ad600. 00010000:00000010:2.1:1775773861.353726:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233f55800. 00010000:00000010:2.1:1775773861.353727:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0afd00. 00010000:00000010:2.1:1775773861.353729:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e298600. 00010000:00000010:2.1:1775773861.353731:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0adf00. 00010000:00000010:2.1:1775773861.353733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233f54600. 00010000:00000010:2.1:1775773861.353734:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0af400. 00010000:00000010:2.1:1775773861.353737:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e298000. 00010000:00000010:2.1:1775773861.353738:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0ad000. 00010000:00000010:2.1:1775773861.353740:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233f54000. 00010000:00000010:2.1:1775773861.353742:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce91700. 00010000:00000010:2.1:1775773861.353743:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233f54a00. 00010000:00000010:2.1:1775773861.353745:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0ad500. 00010000:00000010:2.1:1775773861.353747:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233f55400. 00010000:00000010:2.1:1775773861.353748:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0af800. 00010000:00000010:2.1:1775773861.353750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e298a00. 00010000:00000010:2.1:1775773861.353751:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0adb00. 00010000:00000010:2.1:1775773861.353753:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8f7000. 00010000:00000010:2.1:1775773861.353754:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234adc000. 00010000:00000010:2.1:1775773861.353756:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623476ce00. 00010000:00000010:2.1:1775773861.353758:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0adc00. 00010000:00000010:2.1:1775773861.353759:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233f55c00. 00010000:00000010:2.1:1775773861.353761:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0ad300. 00010000:00000010:2.1:1775773861.353762:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233f54c00. 00010000:00000010:2.1:1775773861.353764:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0ad200. 00010000:00000010:2.1:1775773861.353765:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8f6400. 00010000:00000010:2.1:1775773861.353767:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0afe00. 00010000:00000010:2.1:1775773861.353768:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233f55600. 00010000:00000010:2.1:1775773861.353770:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234adc500. 00010000:00000010:2.1:1775773861.353771:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623476d200. 00010000:00000010:2.1:1775773861.353773:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233961d00. 00010000:00000010:2.1:1775773861.353775:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233f54e00. 00010000:00000010:2.1:1775773861.353777:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233961400. 00010000:00000010:2.1:1775773861.353778:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623476c200. 00010000:00000010:2.1:1775773861.353780:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233961800. 00010000:00000010:2.1:1775773861.353782:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233f54200. 00010000:00000010:2.1:1775773861.353783:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622e0af900. 00010000:00000010:2.1:1775773861.353785:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e299400. 00010000:00000010:2.1:1775773861.353786:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce91a00. 00010000:00000010:2.1:1775773861.353788:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8f7a00. 00010000:00000010:2.1:1775773861.353790:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234adcb00. 00010000:00000010:2.1:1775773861.353792:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8f7600. 00010000:00000010:2.1:1775773861.353794:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d82f00. 00010000:00000010:2.1:1775773861.353796:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8f6c00. 00010000:00000010:2.1:1775773861.353797:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d82000. 00010000:00000010:2.1:1775773861.353799:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e299c00. 00010000:00000010:2.1:1775773861.353801:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233961e00. 00010000:00000010:2.1:1775773861.353802:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233f55a00. 00010000:00000010:2.1:1775773861.353804:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233961900. 00010000:00000010:2.1:1775773861.353805:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233f55000. 00010000:00000010:2.1:1775773861.353807:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233961700. 00010000:00000010:2.1:1775773861.353808:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6233f55e00. 00010000:00000010:2.1:1775773861.353810:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d82500. 00010000:00000010:2.1:1775773861.353811:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622d8f6a00. 00010000:00000010:2.1:1775773861.353813:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce91100. 00010000:00000010:2.1:1775773861.353815:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62267c0400. 00010000:00000010:2.1:1775773861.353817:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d82b00. 00010000:00000010:2.1:1775773861.353818:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e298c00. 00010000:00000010:2.1:1775773861.353820:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233961a00. 00010000:00000010:2.1:1775773861.353821:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62267c0800. 00010000:00000010:2.1:1775773861.353823:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d82c00. 00010000:00000010:2.1:1775773861.353824:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e299600. 00010000:00000010:2.1:1775773861.353826:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234adcc00. 00010000:00000010:2.1:1775773861.353828:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62267c1800. 00010000:00000010:2.1:1775773861.353829:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234adc300. 00010000:00000010:2.1:1775773861.353831:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d7000. 00010000:00000010:2.1:1775773861.353833:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233961100. 00010000:00000010:2.1:1775773861.353834:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62267c0600. 00010000:00000010:2.1:1775773861.353836:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233961600. 00010000:00000010:2.1:1775773861.353838:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62267c0000. 00010000:00000010:2.1:1775773861.353840:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d82300. 00010000:00000010:2.1:1775773861.353842:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e298e00. 00010000:00000010:2.1:1775773861.353844:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d82200. 00010000:00000010:2.1:1775773861.353846:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e298200. 00010000:00000010:2.1:1775773861.353847:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d82d00. 00010000:00000010:2.1:1775773861.353849:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62267c0a00. 00010000:00000010:2.1:1775773861.353851:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233961f00. 00010000:00000010:2.1:1775773861.353852:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62267c1400. 00010000:00000010:2.1:1775773861.353854:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d82400. 00010000:00000010:2.1:1775773861.353855:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e299a00. 00010000:00000010:2.1:1775773861.353857:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d82800. 00010000:00000010:2.1:1775773861.353859:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e299000. 00010000:00000010:2.1:1775773861.353860:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d82e00. 00010000:00000010:2.1:1775773861.353862:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62267c1c00. 00010000:00000010:2.1:1775773861.353863:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233961000. 00010000:00000010:2.1:1775773861.353865:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62267c0c00. 00010000:00000010:2.1:1775773861.353866:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234adc200. 00010000:00000010:2.1:1775773861.353868:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d6e00. 00010000:00000010:2.1:1775773861.353869:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234adcd00. 00010000:00000010:2.1:1775773861.353871:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62267c1600. 00010000:00000010:2.1:1775773861.353872:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d82900. 00010000:00000010:2.1:1775773861.353874:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62267c0e00. 00010000:00000010:2.1:1775773861.353875:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234adc400. 00010000:00000010:2.1:1775773861.353877:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623476c600. 00010000:00000010:2.1:1775773861.353878:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233961500. 00010000:00000010:2.1:1775773861.353880:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62267c0200. 00010000:00000010:2.1:1775773861.353881:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d82700. 00010000:00000010:2.1:1775773861.353883:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e299e00. 00010000:00000010:2.1:1775773861.353884:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce91600. 00010000:00000010:2.1:1775773861.353886:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d6400. 00010000:00000010:2.1:1775773861.353887:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce91f00. 00010000:00000010:2.1:1775773861.353889:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623476da00. 00010000:00000010:2.1:1775773861.353890:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233961b00. 00010000:00000010:2.1:1775773861.353893:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623476de00. 00010000:00000010:2.1:1775773861.353894:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233961c00. 00010000:00000010:2.1:1775773861.353896:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623476c400. 00010000:00000010:2.1:1775773861.353898:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d82a00. 00010000:00000010:2.1:1775773861.353899:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622e299200. 00010000:00000010:2.1:1775773861.353901:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233961300. 00010000:00000010:2.1:1775773861.353903:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62267c1a00. 00010000:00000010:2.1:1775773861.353904:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233961200. 00010000:00000010:2.1:1775773861.353906:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62267c1000. 00010000:00000010:2.1:1775773861.353907:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce91000. 00010000:00000010:2.1:1775773861.353909:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62267c1e00. 00010000:00000010:2.1:1775773861.353911:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233960b00. 00010000:00000010:2.1:1775773861.353912:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62267c1200. 00010000:00000010:2.1:1775773861.353914:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233960c00. 00010000:00000010:2.1:1775773861.353916:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226128e00. 00010000:00000010:2.1:1775773861.353917:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce91500. 00010000:00000010:2.1:1775773861.353919:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f622c1ec200. 00010000:00000010:2.1:1775773861.353920:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233960300. 00010000:00000010:2.1:1775773861.353922:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226128200. 00010000:00000010:2.1:1775773861.353924:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233960200. 00010000:00000010:2.1:1775773861.353925:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226129a00. 00010000:00000010:2.1:1775773861.353927:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233960d00. 00010000:00000010:2.1:1775773861.353928:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226129000. 00010000:00000010:2.1:1775773861.353930:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622ce91b00. 00010000:00000010:2.1:1775773861.353931:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226129e00. 00010000:00000010:2.1:1775773861.353933:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233960400. 00010000:00000010:2.1:1775773861.353934:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621839e800. 00010000:00000010:2.1:1775773861.353936:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb7300. 00010000:00000010:2.1:1775773861.353938:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d6200. 00010000:00000010:2.1:1775773861.353939:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233960800. 00010000:00000010:2.1:1775773861.353941:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226129200. 00010000:00000010:2.1:1775773861.353942:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d82100. 00010000:00000010:2.1:1775773861.353945:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621839f800. 00010000:00000010:2.1:1775773861.353946:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233960e00. 00010000:00000010:2.1:1775773861.353948:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226128400. 00010000:00000010:2.1:1775773861.353950:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6234adc800. 00010000:00000010:2.1:1775773861.353962:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623476dc00. 00010000:00000010:2.1:1775773861.353964:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233960900. 00010000:00000010:2.1:1775773861.353966:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226128800. 00010000:00000010:2.1:1775773861.353968:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6202d82600. 00010000:00000010:2.1:1775773861.353970:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f62265d7a00. 00010000:00000010:2.1:1775773861.353977:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233960700. 00010000:00000010:2.1:1775773861.353979:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226129800. 00010000:00000010:2.1:1775773861.353980:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6206bb7a00. 00010000:00000010:2.1:1775773861.353984:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f623476d000. 00010000:00000010:2.1:1775773861.353986:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622d964200. 00010000:00000010:2.1:1775773861.353988:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226128600. 00010000:00000010:2.1:1775773861.353990:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f622d964d00. 00010000:00000010:2.1:1775773861.353996:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f621839e600. 00010000:00000010:2.1:1775773861.354001:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff8f6233960a00. 00010000:00000010:2.1:1775773861.354004:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff8f6226128000. 00000020:00000001:2.0:1775773862.331329:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773862.331333:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773862.331337:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773863.355278:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773863.355282:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773863.355285:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775773866.299780:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773866.299786:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1775773866.300280:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773866.300282:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1775773866.300491:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773866.300492:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000020:00000001:2.0:1775773866.427302:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773866.427305:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773866.427308:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773867.451314:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773867.451320:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773867.451325:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773868.475328:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773868.475333:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773868.475337:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775773871.419712:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773871.419718:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775773871.420104:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773871.420106:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775773871.420349:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773871.420351:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000020:00000001:2.0:1775773871.547312:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773871.547316:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773871.547320:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773872.571397:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773872.571402:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773872.571405:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773873.595304:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773873.595307:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773873.595310:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775773876.539756:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773876.539761:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775773876.540329:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773876.540332:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775773876.540718:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773876.540721:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000020:00000001:2.0:1775773876.667394:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773876.667401:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773876.667405:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773877.691383:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773877.691387:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773877.691390:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773878.715289:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773878.715293:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773878.715297:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775773881.659246:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773881.659253:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775773881.659256:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773881.659260:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773881.659265:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773881.659272:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 10764 00000400:00000200:2.0:1775773881.659281:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773881.659289:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773881.659292:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773881.659296:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773881.659298:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773881.659300:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773881.659302:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773881.659306:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773881.659307:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775773881.659310:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773881.659312:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775773881.659314:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775773881.659316:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775773881.659317:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773881.659319:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773881.659320:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773881.659321:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773881.659323:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775773881.659325:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773881.659327:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773881.659330:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773881.659332:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773881.659334:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773881.659336:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775773881.659337:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773881.659345:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775773881.659348:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775773881.659351:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773881.659352:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773881.659354:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773881.659357:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773881.659361:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773881.659367:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775773881.659372:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773881.659378:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710222592/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773881.659386:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773881.659390:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773881.659393:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710222592/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773881.659411:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773881.659414:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773881.659416:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775773881.659417:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773881.659419:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773881.659421:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773881.659422:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10764 00000400:00000200:2.0:1775773881.659426:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773881.659429:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773881.659431:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773881.659434:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773881.659435:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773881.659438:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773881.659440:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773881.659443:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773881.659446:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775773881.659448:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773881.659449:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775773881.659452:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775773881.659456:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775773881.659463:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773881.659467:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773881.659470:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773881.659474:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773881.659478:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775773881.659483:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773881.659486:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773881.659488:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773881.659493:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773881.659498:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773881.659502:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773881.659505:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773881.659510:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773881.659512:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773881.659516:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775773881.659518:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773881.659522:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773881.659523:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773881.659526:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773881.659527:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773881.659529:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773881.659534:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775773881.659537:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773881.659542:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710222720/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773881.659554:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773881.659559:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773881.659566:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710222720/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773881.659578:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773881.659583:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773881.659587:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775773881.659590:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773881.659593:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773881.659600:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773881.659605:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10764 00000400:00000200:2.0:1775773881.659612:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773881.659618:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773881.659622:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773881.659629:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773881.659634:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773881.659637:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773881.659641:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773881.659657:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773881.659664:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775773881.659667:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773881.659671:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775773881.659673:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775773881.659676:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775773881.659684:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773881.659686:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773881.659692:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773881.659696:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773881.659709:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775773881.659716:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773881.659721:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773881.659725:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773881.659731:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773881.659736:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773881.659741:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773881.659744:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773881.659751:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773881.659755:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773881.659760:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775773881.659764:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773881.659769:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773881.659770:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773881.659773:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773881.659776:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773881.659779:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773881.659786:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775773881.659793:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773881.659810:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710222848/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773881.659826:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.1:1775773881.659831:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:2.0:1775773881.659832:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775773881.659838:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000100:00000040:2.0:1775773881.659840:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710222848/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773881.659850:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773881.659854:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773881.659858:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773881.659863:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773881.659866:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773881.659870:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (10774) 00000800:00000200:1.1:1775773881.660296:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773881.660298:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775773881.660578:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773881.660580:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000020:00000001:2.0:1775773881.787347:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773881.787352:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773881.787355:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773882.811293:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773882.811297:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773882.811300:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773883.835352:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773883.835355:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773883.835358:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775773886.779795:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773886.779798:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775773886.780092:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773886.780093:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775773886.780304:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773886.780305:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000020:00000001:2.0:1775773886.907304:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773886.907308:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773886.907312:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773887.931359:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773887.931363:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773887.931366:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773888.955316:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773888.955321:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773888.955325:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775773891.899767:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773891.899771:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775773891.900290:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773891.900292:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775773891.900502:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773891.900504:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000020:00000001:2.0:1775773892.027367:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773892.027373:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773892.027376:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773893.051398:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773893.051403:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773893.051406:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775773897.019342:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773897.019356:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775773897.019363:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773897.019369:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773897.019377:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773897.019390:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 10779 00000400:00000200:2.0:1775773897.019407:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773897.019417:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773897.019422:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773897.019426:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773897.019430:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773897.019433:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773897.019436:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773897.019441:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773897.019443:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775773897.019447:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773897.019451:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775773897.019454:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775773897.019457:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775773897.019460:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773897.019462:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773897.019464:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773897.019467:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773897.019470:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775773897.019474:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773897.019478:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773897.019483:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773897.019487:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773897.019491:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773897.019494:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775773897.019496:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773897.019510:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775773897.019514:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775773897.019519:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773897.019522:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773897.019526:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773897.019530:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773897.019536:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773897.019548:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775773897.019557:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773897.019570:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710223744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773897.019584:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773897.019590:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773897.019595:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710223744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773897.019624:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773897.019628:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773897.019633:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775773897.019635:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773897.019636:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773897.019640:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773897.019643:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10779 00000400:00000200:2.0:1775773897.019648:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773897.019652:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773897.019656:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773897.019660:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773897.019664:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773897.019668:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773897.019674:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773897.019677:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773897.019681:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775773897.019685:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773897.019688:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775773897.019694:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775773897.019697:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775773897.019708:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773897.019716:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773897.019723:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773897.019729:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773897.019739:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775773897.019743:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773897.019748:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773897.019754:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773897.019764:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773897.019774:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773897.019780:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773897.019788:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773897.019795:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773897.019798:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773897.019805:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775773897.019814:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773897.019818:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773897.019823:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773897.019828:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773897.019831:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773897.019833:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773897.019840:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775773897.019869:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773897.019883:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710223872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773897.019903:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773897.019908:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773897.019916:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710223872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773897.019926:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773897.019931:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773897.019935:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775773897.019940:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773897.019945:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773897.019954:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773897.019962:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10779 00000400:00000200:2.0:1775773897.019970:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773897.019982:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773897.019987:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773897.019994:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773897.019997:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773897.020004:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773897.020007:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773897.020015:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773897.020018:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775773897.020020:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773897.020022:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775773897.020028:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775773897.020034:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775773897.020039:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773897.020041:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773897.020049:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773897.020054:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773897.020061:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775773897.020069:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773897.020075:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773897.020079:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773897.020085:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773897.020093:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773897.020099:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773897.020104:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773897.020111:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773897.020118:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773897.020125:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775773897.020130:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773897.020204:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773897.020208:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773897.020211:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773897.020217:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773897.020222:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773897.020235:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775773897.020243:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775773897.020252:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00100000:2.0:1775773897.020259:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710224000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1775773897.020260:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1775773897.020287:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773897.020292:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773897.020300:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710224000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773897.020341:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773897.020348:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773897.020355:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773897.020359:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773897.020367:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773897.020375:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (10789) 00000800:00000200:1.1:1775773897.020797:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773897.020800:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775773897.021195:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773897.021198:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1775773897.147394:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773897.147399:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773897.147403:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773898.171287:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773898.171293:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773898.171297:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775773902.139822:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773902.139827:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775773902.140194:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773902.140196:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775773902.140420:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773902.140421:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000020:00000001:2.0:1775773902.267312:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773902.267316:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773902.267319:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773903.291325:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773903.291330:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773903.291333:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775773907.259650:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773907.259655:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775773907.260021:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773907.260023:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775773907.260295:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773907.260298:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000020:00000001:2.0:1775773907.387399:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773907.387403:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773907.387407:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773908.411284:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773908.411289:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773908.411293:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775773912.379862:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773912.379865:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775773912.380198:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773912.380200:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775773912.380386:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773912.380388:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000020:00000001:2.0:1775773912.507346:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773912.507350:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773912.507353:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773913.531356:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773913.531360:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773913.531364:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775773917.499810:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773917.500038:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773917.500194:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:2.0:1775773917.627314:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773917.627321:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773917.627324:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773918.651314:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773918.651317:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773918.651320:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775773922.619260:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773922.619268:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775773922.619271:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773922.619276:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773922.619281:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773922.619289:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 10805 00000400:00000200:2.0:1775773922.619300:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773922.619308:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773922.619311:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773922.619315:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773922.619317:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773922.619320:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773922.619321:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773922.619324:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773922.619326:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775773922.619327:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773922.619330:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775773922.619331:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775773922.619333:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775773922.619335:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773922.619336:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773922.619338:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773922.619339:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773922.619341:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775773922.619343:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773922.619345:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773922.619347:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773922.619350:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773922.619351:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773922.619353:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775773922.619354:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773922.619360:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775773922.619362:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775773922.619365:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773922.619367:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773922.619369:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773922.619371:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773922.619374:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773922.619381:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775773922.619386:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773922.619394:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710225664/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773922.619418:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773922.619424:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773922.619429:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710225664/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773922.619466:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773922.619470:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773922.619473:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775773922.619475:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773922.619477:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773922.619479:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773922.619482:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10805 00000400:00000200:2.0:1775773922.619486:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773922.619535:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773922.619538:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773922.619544:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773922.619550:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773922.619555:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773922.619561:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773922.619566:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773922.619569:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775773922.619572:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773922.619576:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775773922.619581:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775773922.619584:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775773922.619590:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773922.619594:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773922.619599:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773922.619602:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773922.619607:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775773922.619612:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773922.619616:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773922.619619:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773922.619632:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773922.619637:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773922.619640:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773922.619646:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773922.619651:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773922.619655:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773922.619661:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775773922.619664:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773922.619668:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773922.619670:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773922.619673:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773922.619676:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773922.619679:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773922.619685:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775773922.619691:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773922.619702:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710225792/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773922.619718:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773922.619728:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773922.619737:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710225792/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773922.619767:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773922.619771:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773922.619777:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775773922.619781:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773922.619785:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773922.619789:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773922.619796:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10805 00000400:00000200:2.0:1775773922.619806:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773922.619812:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773922.619816:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773922.619820:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773922.619824:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773922.619830:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773922.619835:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773922.619839:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773922.619841:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775773922.619845:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773922.619848:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775773922.619851:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775773922.619853:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775773922.619858:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773922.619861:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773922.619865:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773922.619868:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773922.619873:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775773922.619877:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773922.619882:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773922.619888:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:1.1:1775773922.619888:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:2.0:1775773922.619891:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.1:1775773922.619893:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 02000000:00000001:2.0:1775773922.619895:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773922.619899:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773922.619903:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773922.619908:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773922.619911:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773922.619916:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775773922.619920:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773922.619925:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773922.619928:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773922.619930:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773922.619932:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773922.619935:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773922.619941:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775773922.619946:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773922.619955:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710225920/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773922.619976:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773922.619979:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773922.619987:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710225920/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773922.620007:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773922.620013:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773922.620018:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773922.620023:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773922.620029:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773922.620036:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (10815) 00000800:00000200:1.1:1775773922.620433:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773922.620435:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773922.620732:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773922.620735:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000020:00000001:2.0:1775773922.747319:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773922.747322:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773922.747325:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773923.771330:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773923.771334:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773923.771337:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775773927.739888:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773927.739894:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775773927.740323:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773927.740325:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775773927.740543:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773927.740544:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000020:00000001:2.0:1775773927.867302:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773927.867319:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773927.867324:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773928.891380:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773928.891385:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773928.891390:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775773932.859278:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773932.859288:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775773932.859291:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773932.859297:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773932.859302:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773932.859310:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 10815 00000400:00000200:2.0:1775773932.859321:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773932.859328:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773932.859332:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773932.859336:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773932.859339:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773932.859342:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773932.859345:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773932.859348:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773932.859350:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775773932.859353:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773932.859356:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775773932.859359:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775773932.859361:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775773932.859364:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773932.859366:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773932.859368:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773932.859372:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773932.859374:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775773932.859378:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773932.859381:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773932.859385:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773932.859388:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773932.859392:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773932.859402:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775773932.859405:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773932.859418:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775773932.859422:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775773932.859426:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773932.859428:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773932.859431:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773932.859433:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773932.859437:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773932.859447:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775773932.859453:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773932.859463:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710226432/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773932.859474:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773932.859479:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773932.859484:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710226432/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773932.859507:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773932.859510:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773932.859513:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775773932.859515:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773932.859517:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773932.859519:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773932.859522:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10815 00000400:00000200:2.0:1775773932.859527:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773932.859530:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773932.859532:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773932.859535:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773932.859538:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773932.859541:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773932.859543:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773932.859546:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773932.859548:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775773932.859550:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773932.859553:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775773932.859557:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775773932.859562:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775773932.859571:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773932.859577:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773932.859582:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773932.859588:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773932.859593:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775773932.859602:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773932.859608:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773932.859612:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773932.859620:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773932.859626:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773932.859632:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773932.859636:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773932.859642:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773932.859646:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773932.859653:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775773932.859658:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773932.859664:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773932.859667:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773932.859672:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773932.859675:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773932.859679:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773932.859686:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775773932.859693:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773932.859702:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710226560/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773932.859716:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773932.859720:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773932.859728:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710226560/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773932.859753:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773932.859758:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773932.859765:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775773932.859768:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773932.859771:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773932.859774:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773932.859780:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10815 00000400:00000200:2.0:1775773932.859789:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773932.859797:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000800:00000200:3.1:1775773932.859797:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00080000:2.0:1775773932.859801:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000800:00000200:3.1:1775773932.859802:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000100:00000040:2.0:1775773932.859809:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773932.859814:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773932.859820:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773932.859824:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773932.859830:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773932.859833:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775773932.859837:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773932.859841:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775773932.859845:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775773932.859853:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775773932.859861:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773932.859867:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773932.859872:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773932.859877:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773932.859880:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775773932.859888:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773932.859903:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773932.859907:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773932.859913:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773932.859920:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773932.859926:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773932.859931:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773932.859937:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773932.859942:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773932.859948:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775773932.859953:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773932.859960:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773932.859963:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773932.859968:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773932.859972:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773932.859975:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773932.859983:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775773932.859989:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773932.859999:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710226688/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773932.860021:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773932.860026:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773932.860036:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710226688/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773932.860073:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773932.860079:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773932.860088:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773932.860095:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773932.860105:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773932.860114:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (10825) 00000800:00000200:3.1:1775773932.860278:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773932.860280:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1775773932.860503:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773932.860505:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000020:00000001:2.0:1775773932.987298:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773932.987302:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773932.987305:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773934.011404:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773934.011408:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773934.011411:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775773937.979247:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773937.979267:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775773937.979272:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773937.979277:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773937.979283:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773937.979290:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 10820 00000400:00000200:2.0:1775773937.979299:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773937.979305:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773937.979308:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773937.979312:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773937.979315:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773937.979318:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773937.979320:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773937.979324:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773937.979325:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775773937.979328:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773937.979331:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775773937.979333:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775773937.979336:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775773937.979340:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773937.979342:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773937.979345:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773937.979347:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773937.979350:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775773937.979353:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773937.979356:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773937.979359:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773937.979362:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773937.979365:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773937.979368:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775773937.979370:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773937.979379:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775773937.979382:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775773937.979386:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773937.979388:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773937.979391:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773937.979393:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773937.979397:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773937.979405:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775773937.979412:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773937.979428:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710226816/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773937.979440:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773937.979444:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773937.979449:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710226816/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773937.979470:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773937.979473:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773937.979477:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775773937.979478:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773937.979480:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773937.979483:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773937.979486:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10820 00000400:00000200:2.0:1775773937.979491:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773937.979495:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773937.979498:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773937.979503:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773937.979509:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773937.979512:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773937.979518:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773937.979524:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773937.979527:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775773937.979531:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773937.979535:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775773937.979540:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775773937.979542:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775773937.979547:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773937.979551:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773937.979569:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773937.979574:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773937.979580:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775773937.979587:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773937.979592:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773937.979595:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773937.979602:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773937.979607:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773937.979612:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773937.979618:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773937.979626:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773937.979631:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773937.979641:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775773937.979647:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773937.979653:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773937.979656:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773937.979659:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773937.979662:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773937.979665:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773937.979670:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775773937.979675:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773937.979686:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710226944/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773937.979707:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773937.979713:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773937.979722:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710226944/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773937.979750:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773937.979754:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.1:1775773937.979760:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00080000:2.0:1775773937.979762:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775773937.979765:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:3.1:1775773937.979765:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000100:00080000:2.0:1775773937.979769:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773937.979776:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773937.979781:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10820 00000400:00000200:2.0:1775773937.979788:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773937.979803:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773937.979806:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773937.979809:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773937.979811:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773937.979816:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773937.979820:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773937.979824:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773937.979827:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775773937.979830:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773937.979833:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775773937.979838:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775773937.979842:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775773937.979847:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773937.979852:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773937.979859:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773937.979864:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773937.979871:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775773937.979878:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773937.979883:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773937.979886:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773937.979890:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773937.979896:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773937.979900:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773937.979903:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773937.979908:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773937.979910:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773937.979914:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775773937.979917:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773937.979921:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773937.979923:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773937.979926:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773937.979929:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773937.979932:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773937.979939:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775773937.979947:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773937.979957:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710227072/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773937.979974:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773937.979978:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773937.979988:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710227072/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773937.980012:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773937.980018:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773937.980027:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773937.980033:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773937.980041:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773937.980049:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (10830) 00000800:00000200:3.1:1775773937.980244:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773937.980246:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775773937.980450:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773937.980452:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000020:00000001:2.0:1775773938.107317:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773938.107320:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773938.107322:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773939.131292:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773939.131297:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773939.131301:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775773943.099915:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773943.099919:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775773943.100585:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773943.100588:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1775773943.100874:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773943.100877:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000020:00000001:2.0:1775773943.227283:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773943.227287:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773943.227290:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773944.251333:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773944.251337:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773944.251339:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775773948.219262:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773948.219270:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775773948.219274:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773948.219277:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773948.219281:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773948.219286:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 10830 00000400:00000200:2.0:1775773948.219295:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773948.219300:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773948.219303:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773948.219305:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773948.219308:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773948.219310:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773948.219312:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773948.219315:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773948.219316:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775773948.219319:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773948.219321:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775773948.219329:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775773948.219331:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775773948.219333:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773948.219334:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773948.219335:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773948.219337:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773948.219339:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775773948.219341:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773948.219343:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773948.219345:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773948.219347:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773948.219349:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773948.219351:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775773948.219352:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773948.219359:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775773948.219362:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775773948.219365:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773948.219366:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773948.219368:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773948.219370:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773948.219374:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773948.219379:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775773948.219384:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773948.219391:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710227584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773948.219403:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773948.219407:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773948.219410:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710227584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773948.219430:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773948.219433:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773948.219437:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775773948.219439:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773948.219441:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773948.219444:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773948.219447:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10830 00000400:00000200:2.0:1775773948.219452:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773948.219458:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773948.219463:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773948.219470:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773948.219476:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773948.219481:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773948.219487:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773948.219492:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773948.219495:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775773948.219497:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773948.219500:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775773948.219504:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775773948.219508:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775773948.219512:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773948.219515:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773948.219517:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773948.219520:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773948.219523:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775773948.219525:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773948.219530:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773948.219534:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773948.219539:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773948.219542:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773948.219544:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773948.219548:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773948.219551:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773948.219554:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773948.219557:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775773948.219559:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773948.219563:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773948.219566:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773948.219570:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773948.219572:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773948.219574:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773948.219581:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775773948.219585:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773948.219593:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710227712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773948.219604:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773948.219608:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773948.219614:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710227712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773948.219628:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773948.219631:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773948.219635:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775773948.219637:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773948.219639:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773948.219642:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773948.219647:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10830 00000400:00000200:2.0:1775773948.219663:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773948.219670:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773948.219674:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773948.219680:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773948.219683:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773948.219687:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773948.219689:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773948.219691:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773948.219694:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775773948.219695:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773948.219701:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775773948.219704:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775773948.219709:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:3.1:1775773948.219713:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775773948.219714:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773948.219717:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.1:1775773948.219717:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000100:00000040:2.0:1775773948.219721:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773948.219724:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773948.219728:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775773948.219732:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773948.219735:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773948.219736:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773948.219739:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773948.219742:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773948.219745:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773948.219746:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773948.219750:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773948.219752:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773948.219758:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775773948.219760:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773948.219764:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773948.219766:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773948.219770:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773948.219772:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773948.219773:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773948.219776:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775773948.219780:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773948.219786:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710227840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773948.219809:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773948.219810:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773948.219819:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710227840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773948.219847:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773948.219850:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773948.219854:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773948.219857:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773948.219860:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773948.219867:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (10840) 00000800:00000200:3.1:1775773948.220125:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773948.220126:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1775773948.220365:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773948.220367:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000020:00000001:2.0:1775773948.347306:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773948.347309:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773948.347312:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773949.371351:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773949.371356:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773949.371360:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775773953.340117:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773953.340124:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775773953.340799:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773953.340801:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1775773953.341127:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773953.341129:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000020:00000001:2.0:1775773953.467433:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773953.467438:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773953.467443:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773954.491328:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773954.491333:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773954.491337:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775773958.459753:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773958.459756:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775773958.460223:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773958.460226:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1775773958.460550:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773958.460553:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000020:00000001:2.0:1775773958.587340:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773958.587345:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773958.587348:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773959.611298:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773959.611302:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773959.611304:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775773963.580022:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773963.580034:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775773963.580397:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773963.580399:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775773963.580580:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773963.580582:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1775773963.707373:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773963.707377:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773963.707380:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773964.731328:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773964.731334:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773964.731337:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775773968.699255:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773968.699265:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775773968.699268:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773968.699272:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773968.699278:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773968.699286:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 10851 00000400:00000200:2.0:1775773968.699295:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773968.699300:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773968.699303:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773968.699306:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773968.699309:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773968.699311:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773968.699312:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773968.699315:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773968.699317:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775773968.699327:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773968.699329:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775773968.699331:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775773968.699333:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775773968.699335:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773968.699336:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773968.699338:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773968.699340:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773968.699342:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775773968.699345:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773968.699346:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773968.699349:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773968.699351:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773968.699353:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773968.699354:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775773968.699356:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773968.699363:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775773968.699366:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775773968.699370:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773968.699371:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773968.699374:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773968.699376:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773968.699380:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773968.699386:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775773968.699391:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773968.699398:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710229120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773968.699407:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773968.699410:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773968.699413:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710229120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773968.699441:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773968.699444:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773968.699448:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775773968.699450:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773968.699452:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773968.699455:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773968.699459:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10851 00000400:00000200:2.0:1775773968.699464:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773968.699469:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773968.699473:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773968.699480:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773968.699485:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773968.699490:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773968.699496:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773968.699502:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773968.699503:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775773968.699505:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773968.699506:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775773968.699508:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775773968.699509:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775773968.699511:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773968.699514:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773968.699517:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773968.699522:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773968.699526:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775773968.699532:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773968.699537:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773968.699538:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773968.699543:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773968.699550:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773968.699553:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773968.699557:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773968.699563:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773968.699568:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773968.699574:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775773968.699579:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773968.699583:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773968.699587:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773968.699605:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773968.699610:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773968.699615:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773968.699623:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775773968.699630:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773968.699641:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710229248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773968.699658:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773968.699660:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773968.699667:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710229248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773968.699682:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773968.699685:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773968.699690:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775773968.699693:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773968.699696:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773968.699700:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773968.699704:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10851 00000400:00000200:2.0:1775773968.699720:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773968.699727:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773968.699731:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773968.699736:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773968.699739:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773968.699742:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773968.699744:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773968.699748:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773968.699749:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775773968.699750:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773968.699752:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775773968.699756:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775773968.699760:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775773968.699765:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773968.699768:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.1:1775773968.699769:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775773968.699772:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:3.1:1775773968.699774:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000100:00000001:2.0:1775773968.699775:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773968.699780:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775773968.699785:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773968.699788:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773968.699790:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773968.699792:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773968.699797:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773968.699799:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773968.699802:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773968.699805:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773968.699807:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773968.699811:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775773968.699814:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773968.699818:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773968.699820:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773968.699825:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773968.699827:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773968.699830:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773968.699836:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775773968.699843:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773968.699852:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710229376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773968.699870:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773968.699873:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773968.699881:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710229376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773968.699900:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773968.699903:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773968.699912:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773968.699916:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773968.699927:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773968.699934:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (10861) 00000800:00000200:3.1:1775773968.700131:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773968.700149:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775773968.700370:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773968.700373:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000020:00000001:2.0:1775773968.827292:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773968.827296:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773968.827300:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773969.851461:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773969.851467:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773969.851470:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775773973.819250:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773973.819262:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775773973.819268:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773973.819274:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773973.819281:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773973.819290:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 10856 00000400:00000200:2.0:1775773973.819301:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773973.819308:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773973.819311:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773973.819314:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773973.819318:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773973.819321:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773973.819324:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773973.819328:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773973.819330:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775773973.819333:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773973.819336:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775773973.819339:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775773973.819341:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775773973.819345:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773973.819347:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773973.819349:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773973.819351:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773973.819355:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775773973.819358:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773973.819360:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773973.819364:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773973.819367:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773973.819372:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773973.819374:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775773973.819377:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773973.819385:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775773973.819388:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775773973.819393:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773973.819395:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773973.819398:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773973.819400:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773973.819405:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773973.819413:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775773973.819419:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773973.819428:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710229504/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773973.819439:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773973.819443:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773973.819448:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710229504/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773973.819475:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773973.819478:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773973.819482:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775773973.819484:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773973.819486:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773973.819488:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773973.819491:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10856 00000400:00000200:2.0:1775773973.819494:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773973.819497:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773973.819515:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773973.819520:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773973.819523:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773973.819526:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773973.819528:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773973.819531:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773973.819532:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775773973.819536:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773973.819539:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775773973.819543:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775773973.819546:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775773973.819552:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773973.819557:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773973.819561:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773973.819565:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773973.819570:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775773973.819576:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773973.819581:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773973.819585:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773973.819593:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773973.819598:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773973.819602:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773973.819608:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773973.819614:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773973.819619:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773973.819626:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775773973.819630:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773973.819634:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773973.819636:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773973.819641:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773973.819644:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773973.819648:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773973.819656:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775773973.819662:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773973.819673:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710229632/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773973.819686:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773973.819690:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773973.819697:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710229632/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773973.819718:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773973.819723:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773973.819731:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775773973.819733:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773973.819735:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773973.819742:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773973.819748:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10856 00000400:00000200:2.0:1775773973.819757:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773973.819763:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773973.819766:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773973.819774:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773973.819779:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.1:1775773973.819780:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775773973.819784:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:3.1:1775773973.819785:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000100:00000001:2.0:1775773973.819790:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773973.819796:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773973.819809:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775773973.819817:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773973.819820:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775773973.819824:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775773973.819829:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775773973.819834:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773973.819839:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773973.819844:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773973.819847:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773973.819851:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775773973.819857:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773973.819863:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773973.819868:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773973.819872:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773973.819876:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773973.819879:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773973.819882:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773973.819889:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773973.819893:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773973.819900:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775773973.819905:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773973.819908:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773973.819911:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773973.819914:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773973.819917:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773973.819920:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773973.819925:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775773973.819930:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773973.819938:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710229760/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773973.819949:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773973.819954:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773973.819964:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710229760/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773973.819990:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773973.819995:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773973.820003:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773973.820009:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773973.820017:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773973.820026:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (10866) 00000800:00000200:3.1:1775773973.820236:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773973.820237:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775773973.820458:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773973.820460:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000020:00000001:2.0:1775773973.947327:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773973.947332:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773973.947336:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773974.971377:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773974.971381:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773974.971384:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775773978.939223:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773978.939232:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775773978.939236:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773978.939242:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773978.939248:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773978.939257:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 10861 00000400:00000200:2.0:1775773978.939271:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773978.939282:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773978.939286:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773978.939290:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773978.939294:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773978.939297:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773978.939300:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773978.939316:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773978.939317:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775773978.939321:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773978.939324:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775773978.939326:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775773978.939329:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775773978.939332:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773978.939334:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773978.939336:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773978.939338:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773978.939341:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775773978.939345:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773978.939347:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773978.939351:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773978.939353:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773978.939355:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773978.939357:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775773978.939358:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773978.939365:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775773978.939368:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775773978.939371:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773978.939372:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773978.939374:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773978.939376:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773978.939379:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773978.939387:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775773978.939392:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773978.939399:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710229888/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773978.939407:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773978.939424:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773978.939430:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710229888/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773978.939458:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773978.939461:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773978.939464:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775773978.939465:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773978.939466:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773978.939468:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773978.939470:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10861 00000400:00000200:2.0:1775773978.939472:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773978.939475:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773978.939477:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773978.939479:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773978.939481:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773978.939483:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773978.939485:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773978.939487:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773978.939488:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775773978.939489:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773978.939491:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775773978.939492:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775773978.939493:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775773978.939496:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773978.939499:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773978.939502:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773978.939504:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773978.939507:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775773978.939512:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773978.939516:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773978.939520:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773978.939526:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773978.939530:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773978.939533:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773978.939538:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773978.939542:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773978.939546:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773978.939552:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775773978.939554:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773978.939558:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773978.939560:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773978.939562:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773978.939565:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773978.939567:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773978.939573:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775773978.939577:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773978.939584:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710230016/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773978.939593:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773978.939595:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773978.939598:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710230016/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773978.939606:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773978.939609:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773978.939614:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775773978.939616:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773978.939618:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773978.939621:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773978.939626:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10861 00000400:00000200:2.0:1775773978.939634:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773978.939638:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773978.939642:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773978.939648:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773978.939651:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773978.939654:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773978.939659:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773978.939661:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773978.939663:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775773978.939665:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773978.939668:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775773978.939671:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775773978.939672:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775773978.939676:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773978.939678:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773978.939680:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773978.939682:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773978.939685:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775773978.939691:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773978.939693:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773978.939696:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773978.939698:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773978.939701:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773978.939704:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773978.939706:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773978.939709:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773978.939713:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773978.939716:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775773978.939720:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773978.939723:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773978.939725:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773978.939728:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773978.939728:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773978.939730:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773978.939732:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775773978.939737:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773978.939744:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710230144/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773978.939763:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773978.939765:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773978.939773:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710230144/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.1:1775773978.939785:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773978.939790:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000100:00000001:2.0:1775773978.939793:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773978.939798:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773978.939806:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773978.939809:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773978.939812:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773978.939817:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (10871) 00000800:00000200:3.1:1775773978.940202:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773978.940204:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775773978.940426:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773978.940428:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000020:00000001:2.0:1775773979.067358:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773979.067362:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773979.067365:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773980.091257:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773980.091260:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773980.091263:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775773984.059256:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773984.059264:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775773984.059267:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773984.059270:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773984.059274:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773984.059286:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 10866 00000400:00000200:2.0:1775773984.059298:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773984.059304:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773984.059306:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773984.059311:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773984.059315:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773984.059320:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773984.059336:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773984.059340:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773984.059341:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775773984.059344:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773984.059347:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775773984.059351:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775773984.059355:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775773984.059360:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773984.059363:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773984.059367:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773984.059369:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773984.059371:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775773984.059373:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773984.059375:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773984.059378:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773984.059380:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773984.059383:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773984.059384:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775773984.059386:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773984.059392:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775773984.059394:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775773984.059398:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773984.059399:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773984.059401:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773984.059404:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773984.059407:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773984.059413:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775773984.059419:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773984.059427:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710230272/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773984.059435:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773984.059439:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773984.059442:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710230272/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773984.059459:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773984.059461:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773984.059464:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775773984.059465:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773984.059466:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773984.059468:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773984.059470:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10866 00000400:00000200:2.0:1775773984.059473:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773984.059475:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773984.059477:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773984.059480:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773984.059482:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773984.059484:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773984.059486:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773984.059488:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773984.059489:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775773984.059491:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773984.059492:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775773984.059495:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775773984.059498:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775773984.059502:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773984.059504:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773984.059508:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773984.059509:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773984.059513:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775773984.059518:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773984.059527:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773984.059532:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773984.059537:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773984.059542:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773984.059544:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773984.059549:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773984.059553:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773984.059557:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773984.059561:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775773984.059564:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773984.059568:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773984.059570:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773984.059572:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773984.059574:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773984.059575:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773984.059579:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775773984.059584:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773984.059590:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710230400/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773984.059600:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773984.059602:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773984.059609:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710230400/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773984.059620:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773984.059625:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773984.059628:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775773984.059630:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773984.059631:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773984.059636:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773984.059641:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10866 00000400:00000200:2.0:1775773984.059648:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773984.059654:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773984.059657:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773984.059661:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773984.059667:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773984.059668:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773984.059670:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773984.059674:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773984.059676:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775773984.059677:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773984.059679:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775773984.059684:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775773984.059688:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775773984.059696:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773984.059702:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773984.059708:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773984.059714:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773984.059720:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775773984.059728:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773984.059734:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773984.059736:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773984.059742:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773984.059747:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773984.059749:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773984.059751:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773984.059755:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773984.059756:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773984.059759:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775773984.059762:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773984.059767:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773984.059768:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773984.059772:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773984.059774:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:3.1:1775773984.059775:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:2.0:1775773984.059777:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773984.059782:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775773984.059784:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773984.059788:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710230528/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773984.059796:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773984.059798:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773984.059803:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710230528/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773984.059816:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773984.059821:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773984.059824:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773984.059826:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773984.059830:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773984.059834:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (10876) 00000800:00000200:3.1:1775773984.059970:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775773984.060104:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:2.0:1775773984.187301:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773984.187304:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773984.187307:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773985.211354:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773985.211357:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773985.211360:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775773989.180017:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773989.180024:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773989.180550:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773989.180552:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775773989.180897:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773989.180900:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000020:00000001:2.0:1775773989.307326:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773989.307330:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773989.307333:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773990.331424:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773990.331429:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773990.331432:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775773994.299264:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773994.299272:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775773994.299276:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773994.299280:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773994.299285:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773994.299293:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 10876 00000400:00000200:2.0:1775773994.299302:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773994.299307:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773994.299310:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773994.299313:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773994.299316:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773994.299317:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773994.299319:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773994.299322:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773994.299324:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775773994.299326:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773994.299328:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775773994.299329:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775773994.299331:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775773994.299333:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773994.299335:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773994.299336:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773994.299337:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773994.299339:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775773994.299341:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773994.299343:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773994.299345:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773994.299347:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773994.299349:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773994.299350:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775773994.299352:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773994.299360:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775773994.299362:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775773994.299365:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773994.299367:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773994.299369:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773994.299371:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773994.299374:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773994.299381:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775773994.299386:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773994.299393:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710231040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773994.299401:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773994.299405:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773994.299408:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710231040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773994.299426:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773994.299429:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773994.299431:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775773994.299432:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773994.299433:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773994.299435:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773994.299437:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10876 00000400:00000200:2.0:1775773994.299439:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773994.299442:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773994.299444:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773994.299446:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773994.299448:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773994.299450:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773994.299452:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773994.299455:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773994.299481:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775773994.299483:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773994.299488:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775773994.299493:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775773994.299498:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775773994.299503:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773994.299505:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773994.299509:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773994.299512:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773994.299515:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775773994.299523:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773994.299530:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773994.299537:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773994.299543:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773994.299547:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773994.299571:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773994.299575:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773994.299578:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773994.299589:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773994.299594:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775773994.299602:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773994.299605:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773994.299609:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773994.299611:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773994.299612:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773994.299614:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773994.299620:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775773994.299627:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773994.299634:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710231168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773994.299640:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773994.299643:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773994.299650:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710231168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773994.299685:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773994.299694:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773994.299699:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775773994.299706:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773994.299710:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773994.299715:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773994.299725:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10876 00000400:00000200:2.0:1775773994.299743:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775773994.299756:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773994.299764:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773994.299775:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773994.299782:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773994.299785:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773994.299788:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775773994.299792:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773994.299794:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775773994.299799:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773994.299818:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775773994.299825:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775773994.299828:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775773994.299839:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773994.299847:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773994.299855:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775773994.299865:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775773994.299872:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775773994.299884:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775773994.299895:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773994.299897:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773994.299903:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773994.299906:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773994.299922:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773994.299927:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773994.299938:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773994.299940:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773994.299944:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775773994.299947:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775773994.299951:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773994.299959:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773994.299963:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773994.299967:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773994.299974:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775773994.299990:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773994.299995:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 02000000:00000010:2.0:1775773994.299996:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775773994.300002:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773994.300014:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710231296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773994.300020:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773994.300026:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773994.300037:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710231296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773994.300062:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773994.300068:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773994.300079:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773994.300090:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775773994.300102:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775773994.300122:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (10887) 00000800:00000200:1.1:1775773994.300478:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773994.300480:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1775773994.300692:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773994.300694:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000020:00000001:2.0:1775773994.427344:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773994.427348:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773994.427351:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775773995.451328:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773995.451332:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773995.451336:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775773999.420155:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773999.420159:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1775773999.420767:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773999.420770:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1775773999.421077:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775773999.421079:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000020:00000001:2.0:1775773999.547317:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773999.547324:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773999.547329:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774000.571339:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774000.571344:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774000.571347:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774004.539725:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774004.539732:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775774004.540263:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774004.540265:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775774004.540447:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774004.540449:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000020:00000001:2.0:1775774004.667353:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774004.667356:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774004.667359:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774005.691292:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774005.691297:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774005.691300:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774009.659722:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774009.659728:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775774009.660287:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774009.660290:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775774009.660580:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774009.660582:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000020:00000001:2.0:1775774009.787322:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774009.787326:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774009.787330:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774010.811296:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774010.811299:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774010.811301:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774014.780070:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774014.780076:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775774014.780705:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774014.780708:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775774014.781107:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774014.781110:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000020:00000001:2.0:1775774014.907334:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774014.907341:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774014.907347:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774015.931447:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774015.931453:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774015.931458:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774019.899815:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774019.899821:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775774019.900244:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774019.900246:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775774019.900451:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774019.900453:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000020:00000001:2.0:1775774020.027376:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774020.027382:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774020.027386:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774021.051327:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774021.051333:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774021.051338:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774025.019971:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774025.019978:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774025.020657:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774025.020659:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774025.020911:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774025.020914:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000020:00000001:2.0:1775774025.147366:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774025.147374:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774025.147379:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774026.171349:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774026.171353:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774026.171356:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774030.139296:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774030.139305:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774030.139308:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774030.139315:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774030.139320:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774030.139330:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 10912 00000400:00000200:2.0:1775774030.139340:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774030.139348:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774030.139351:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774030.139354:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774030.139357:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774030.139359:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774030.139361:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774030.139364:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774030.139365:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774030.139367:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774030.139370:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774030.139371:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774030.139373:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774030.139375:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774030.139376:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774030.139377:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774030.139379:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774030.139381:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774030.139383:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774030.139385:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774030.139387:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774030.139391:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774030.139393:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774030.139394:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774030.139396:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774030.139404:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775774030.139415:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774030.139419:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774030.139420:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774030.139422:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774030.139424:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774030.139427:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774030.139434:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775774030.139438:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774030.139445:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710233728/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774030.139453:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774030.139457:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774030.139460:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710233728/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774030.139478:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774030.139481:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774030.139483:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774030.139484:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774030.139485:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774030.139487:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774030.139489:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10912 00000400:00000200:2.0:1775774030.139491:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774030.139494:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774030.139495:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774030.139498:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774030.139500:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774030.139502:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774030.139504:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774030.139506:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774030.139507:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774030.139508:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774030.139509:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774030.139511:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774030.139512:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774030.139514:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774030.139527:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774030.139530:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774030.139534:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774030.139537:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774030.139542:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774030.139543:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774030.139547:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774030.139553:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774030.139557:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774030.139560:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774030.139564:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774030.139570:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774030.139574:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774030.139578:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775774030.139583:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774030.139586:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774030.139589:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774030.139590:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774030.139593:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774030.139596:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774030.139601:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775774030.139606:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774030.139618:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710233856/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774030.139630:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774030.139635:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774030.139644:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710233856/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774030.139669:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774030.139673:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774030.139682:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774030.139684:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774030.139686:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774030.139692:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774030.139698:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10912 00000400:00000200:2.0:1775774030.139707:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774030.139716:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774030.139719:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774030.139722:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774030.139727:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774030.139734:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774030.139740:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774030.139745:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774030.139748:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774030.139750:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774030.139755:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774030.139761:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774030.139767:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774030.139775:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774030.139778:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774030.139786:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774030.139792:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774030.139797:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000800:00000200:1.1:1775774030.139797:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774030.139801:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00080000:2.0:1775774030.139804:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774030.139809:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774030.139813:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774030.139817:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774030.139820:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774030.139821:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774030.139823:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774030.139829:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774030.139830:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774030.139835:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775774030.139839:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774030.139841:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774030.139842:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774030.139846:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774030.139848:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774030.139850:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774030.139854:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775774030.139858:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774030.139865:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710233984/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774030.139890:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774030.139892:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774030.139894:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710233984/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774030.139904:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774030.139908:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774030.139913:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774030.139919:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774030.139927:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774030.139933:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (10922) 00000800:00000200:1.1:1775774030.140198:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774030.140200:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774030.140447:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774030.140449:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1775774030.267393:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774030.267398:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774030.267401:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774031.291350:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774031.291356:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774031.291360:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774035.259307:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774035.259324:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774035.259328:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774035.259335:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774035.259347:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774035.259355:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 10917 00000400:00000200:2.0:1775774035.259367:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774035.259373:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774035.259376:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774035.259379:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774035.259382:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774035.259384:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774035.259386:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774035.259390:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774035.259392:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774035.259394:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774035.259397:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774035.259399:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774035.259401:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774035.259404:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774035.259405:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774035.259409:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774035.259410:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774035.259412:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774035.259416:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774035.259420:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774035.259423:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774035.259440:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774035.259444:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774035.259447:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774035.259451:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774035.259463:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775774035.259468:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774035.259473:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774035.259475:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774035.259478:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774035.259482:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774035.259487:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774035.259498:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775774035.259506:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774035.259517:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710234112/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774035.259526:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774035.259531:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774035.259535:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710234112/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774035.259553:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774035.259556:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774035.259559:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774035.259561:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774035.259562:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774035.259564:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774035.259566:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10917 00000400:00000200:2.0:1775774035.259570:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774035.259572:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774035.259574:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774035.259576:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774035.259578:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774035.259579:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774035.259582:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774035.259584:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774035.259586:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774035.259587:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774035.259590:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774035.259592:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774035.259593:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774035.259595:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774035.259597:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774035.259599:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774035.259601:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774035.259603:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774035.259605:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774035.259607:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774035.259609:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774035.259622:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774035.259625:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774035.259626:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774035.259630:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774035.259634:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774035.259640:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774035.259649:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775774035.259655:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774035.259663:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774035.259667:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774035.259677:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774035.259679:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774035.259682:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774035.259687:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775774035.259692:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774035.259699:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710234240/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774035.259715:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774035.259721:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774035.259727:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710234240/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774035.259747:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774035.259752:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774035.259769:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774035.259774:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774035.259777:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774035.259783:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774035.259789:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 10917 00000400:00000200:2.0:1775774035.259798:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774035.259807:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774035.259809:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774035.259813:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774035.259816:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774035.259818:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774035.259820:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774035.259823:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774035.259824:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774035.259834:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774035.259837:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774035.259842:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774035.259851:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774035.259869:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774035.259877:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774035.259889:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774035.259895:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774035.259901:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774035.259909:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774035.259917:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774035.259921:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774035.259925:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774035.259932:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774035.259937:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774035.259941:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774035.259947:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774035.259954:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774035.259959:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775774035.259964:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774035.259973:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774035.259978:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774035.259983:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774035.259990:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774035.259994:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774035.260004:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775774035.260012:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774035.260023:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710234368/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774035.260052:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774035.260056:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774035.260063:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710234368/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774035.260078:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774035.260081:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774035.260086:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774035.260094:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774035.260098:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774035.260106:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (10927) 00000800:00000200:1.1:1775774035.260113:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774035.260118:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774035.260605:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774035.260607:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774035.260933:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774035.260936:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000020:00000001:2.0:1775774035.387322:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774035.387326:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774035.387329:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774036.411306:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774036.411310:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774036.411312:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775774040.380018:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774040.380023:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775774040.380601:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774040.380603:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1775774040.380834:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774040.380836:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000020:00000001:2.0:1775774040.507398:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774040.507404:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774040.507409:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774041.531371:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774041.531376:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774041.531381:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775774045.499845:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774045.499849:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775774045.500227:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774045.500228:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1775774045.500538:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774045.500541:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000020:00000001:2.0:1775774045.627366:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774045.627371:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774045.627375:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774046.651576:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774046.651580:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774046.651583:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775774050.619954:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774050.620236:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774050.620333:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:2.0:1775774050.747366:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774050.747369:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774050.747372:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774051.771360:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774051.771363:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774051.771366:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775774053.691528:0:2399:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:2.0:1775774053.691581:0:2399:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1775774053.691594:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774053.691597:0:2399:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:2.0:1775774053.691603:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774053.691609:0:2399:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:2.0:1775774053.691615:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775774053.691619:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101298e00. 00000400:00000200:2.0:1775774053.691628:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775774053.691635:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775774053.691643:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62079b2220 00000400:00000010:2.0:1775774053.691648:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62079b2220. 00000100:00000001:2.0:1775774053.691657:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775774053.691674:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6234919c00 x1862022710235264/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775774075 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774053.691721:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775774053.691726:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234919c00 x1862022710235264/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775774075 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774053.691735:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774053.691738:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775774053.691755:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775774053.691759:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101298100. 00000400:00000200:2.0:1775774053.691765:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775774053.691769:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775774053.691773:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62079b2198 00000400:00000010:2.0:1775774053.691776:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62079b2198. 00000100:00000001:2.0:1775774053.691779:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775774053.691786:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f623491a300 x1862022710235392/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775774075 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774053.691808:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775774053.691815:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491a300 x1862022710235392/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775774075 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774053.691824:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774053.691828:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775774053.691855:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775774053.691862:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101298500. 00000400:00000200:2.0:1775774053.691872:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775774053.691879:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775774053.691884:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62079b2088 00000400:00000010:2.0:1775774053.691892:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62079b2088. 00000100:00000001:2.0:1775774053.691898:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775774053.691925:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f623491a680 x1862022710235520/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775774075 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774053.691954:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775774053.691964:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491a680 x1862022710235520/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775774075 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774053.692012:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774053.692022:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1775774055.739353:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774055.739361:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774055.739365:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774055.739371:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774055.739378:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774055.739385:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774055.739390:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774055.739394:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774055.739398:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774055.740080:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774055.740090:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774055.740712:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774055.740716:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774055.741018:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774055.741021:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000020:00000001:2.0:1775774055.867295:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774055.867308:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774055.867312:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774056.763436:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774056.763445:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774056.763448:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774056.763455:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774056.763460:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774056.763465:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774056.763469:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774056.763472:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774056.763476:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774056.891343:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774056.891346:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774056.891349:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774057.787239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774057.787245:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774057.787247:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774057.787254:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774057.787258:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774057.787262:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774057.787265:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774057.787268:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774057.787271:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774058.811192:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774058.811197:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774058.811199:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774058.811203:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774058.811205:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774058.811208:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774058.811210:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774058.811212:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774058.811214:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774059.835207:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774059.835213:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774059.835214:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774059.835218:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774059.835221:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774059.835224:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774059.835226:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774059.835228:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774059.835230:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774060.859215:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774060.859220:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774060.859223:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774060.859227:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774060.859232:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774060.859236:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774060.859240:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774060.859242:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774060.859245:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774060.859796:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774060.859799:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774060.860244:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774060.860247:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774060.860543:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774060.860546:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000020:00000001:2.0:1775774060.987365:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774060.987369:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774060.987372:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774062.011334:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774062.011338:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774062.011343:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774063.931273:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774063.931280:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774063.931282:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774063.931287:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774063.931291:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774063.931295:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774063.931298:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774063.931300:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774063.931303:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774064.955271:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774064.955279:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774064.955281:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774064.955288:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774064.955292:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774064.955296:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774064.955299:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774064.955303:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774064.955306:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774065.979237:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774065.979242:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774065.979245:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774065.979249:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774065.979254:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774065.979257:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774065.979259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774065.979262:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774065.979265:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774065.979822:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774065.979826:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774065.980319:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774065.980322:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774065.980635:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774065.980638:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1775774067.003238:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774067.003244:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774067.003246:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774067.003251:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774067.003254:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774067.003257:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774067.003260:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774067.003261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774067.003263:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774067.131361:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774067.131365:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774067.131367:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774068.027208:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774068.027214:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774068.027215:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774068.027219:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774068.027223:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774068.027226:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774068.027228:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774068.027229:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774068.027232:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774069.051311:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774069.051319:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774069.051321:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774069.051326:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774069.051329:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774069.051332:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774069.051334:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774069.051336:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774069.051338:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775774069.883264:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774069.883272:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774069.883277:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000100:00000001:2.0:1775774070.075198:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774070.075203:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774070.075205:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774070.075210:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774070.075213:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774070.075216:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774070.075218:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774070.075220:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774070.075222:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774071.099258:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774071.099266:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774071.099269:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774071.099275:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774071.099281:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774071.099288:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774071.099293:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774071.099297:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774071.099300:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774071.099897:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774071.099904:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774071.100319:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774071.100322:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774071.100621:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774071.100623:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000100:00000001:2.0:1775774072.123221:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774072.123229:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774072.123232:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774072.123238:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774072.123243:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774072.123246:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774072.123249:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774072.123252:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774072.123255:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774072.251372:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774072.251375:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774072.251378:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774073.147256:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774073.147262:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774073.147263:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774073.147268:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774073.147271:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774073.147274:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774073.147276:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774073.147278:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774073.147280:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774074.171274:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774074.171284:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774074.171286:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774074.171293:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774074.171298:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774074.171301:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774074.171304:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774074.171306:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774074.171309:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774075.195290:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774075.195297:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774075.195299:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774075.195305:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774075.195310:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774075.195316:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774075.195319:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774075.195322:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774075.195325:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774076.219273:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774076.219282:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774076.219286:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774076.219295:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774076.219304:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774076.219310:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774076.219317:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774076.219321:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774076.219332:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774076.219912:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774076.219916:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775774076.220414:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774076.220417:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775774076.220701:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774076.220703:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000100:00000001:2.0:1775774077.243280:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774077.243288:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774077.243290:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774077.243296:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774077.243299:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774077.243304:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774077.243306:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774077.243309:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774077.243312:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774077.371341:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774077.371346:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774077.371350:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774078.267258:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774078.267269:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774078.267272:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774078.267280:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774078.267285:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774078.267290:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774078.267293:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774078.267297:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774078.267301:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774079.291301:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774079.291311:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774079.291313:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774079.291322:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774079.291328:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774079.291333:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774079.291337:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774079.291341:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774079.291344:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774080.315205:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774080.315212:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774080.315214:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774080.315220:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774080.315223:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774080.315227:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774080.315230:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774080.315232:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774080.315235:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774081.339268:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774081.339276:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774081.339279:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774081.339285:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774081.339290:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774081.339296:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774081.339299:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774081.339302:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774081.339306:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774081.339922:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774081.339925:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775774081.340273:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774081.340275:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775774081.340541:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774081.340543:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000100:00000001:2.0:1775774082.363273:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774082.363282:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774082.363284:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774082.363291:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774082.363295:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774082.363301:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774082.363305:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774082.363308:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774082.363311:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774082.491400:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774082.491404:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774082.491408:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774083.387259:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774083.387271:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774083.387274:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774083.387283:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774083.387288:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774083.387293:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774083.387296:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774083.387299:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774083.387303:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774084.411285:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774084.411293:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774084.411295:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774084.411301:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774084.411305:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774084.411309:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774084.411313:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774084.411315:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774084.411318:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774085.435222:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774085.435229:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774085.435231:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774085.435236:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774085.435240:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774085.435244:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774085.435247:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774085.435250:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774085.435252:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774086.459284:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774086.459292:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774086.459294:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774086.459300:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774086.459304:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774086.459307:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774086.459310:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774086.459313:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774086.459317:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774086.460956:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774086.460962:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775774086.461404:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774086.461406:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775774086.461683:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774086.461685:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000100:00000001:2.0:1775774087.483210:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774087.483215:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774087.483217:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774087.483221:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774087.483225:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774087.483228:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774087.483230:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774087.483232:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774087.483234:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774087.611312:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774087.611317:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774087.611321:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774088.507235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774088.507241:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774088.507243:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774088.507247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774088.507251:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774088.507254:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774088.507256:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774088.507258:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774088.507259:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774089.531207:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774089.531213:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774089.531215:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774089.531219:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774089.531222:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774089.531224:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774089.531226:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774089.531228:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774089.531230:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774090.555235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774090.555241:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774090.555242:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774090.555247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774090.555251:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774090.555253:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774090.555255:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774090.555257:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774090.555259:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774091.579207:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774091.579214:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774091.579215:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774091.579220:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774091.579223:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774091.579226:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774091.579229:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774091.579231:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774091.579233:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774091.579810:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774091.579814:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774091.580269:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774091.580271:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774091.580621:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774091.580624:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1775774092.603241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774092.603248:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774092.603249:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774092.603255:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774092.603260:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774092.603264:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774092.603266:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774092.603269:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774092.603272:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774092.731315:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774092.731318:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774092.731321:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774093.627189:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774093.627195:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774093.627197:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774093.627201:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774093.627210:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774093.627213:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774093.627215:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774093.627217:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774093.627219:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774094.651230:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774094.651236:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774094.651238:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774094.651243:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774094.651246:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774094.651249:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774094.651251:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774094.651253:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774094.651255:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774095.675222:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774095.675227:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774095.675229:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774095.675239:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774095.675241:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774095.675244:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774095.675246:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774095.675248:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774095.675249:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774096.699253:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774096.699259:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774096.699262:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774096.699270:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774096.699277:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774096.699282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774096.699287:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774096.699291:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774096.699295:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774096.699807:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774096.699811:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774096.700333:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774096.700335:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774096.700652:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774096.700654:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1775774097.723215:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774097.723222:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774097.723224:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774097.723231:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774097.723235:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774097.723239:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774097.723242:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774097.723244:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774097.723247:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774097.851403:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774097.851408:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774097.851411:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774098.747249:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774098.747254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774098.747255:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774098.747259:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774098.747263:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774098.747266:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774098.747268:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774098.747270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774098.747272:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774099.771245:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774099.771250:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774099.771252:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774099.771256:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774099.771259:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774099.771263:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774099.771265:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774099.771267:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774099.771268:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774100.795248:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774100.795256:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774100.795258:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774100.795264:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774100.795270:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774100.795274:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774100.795277:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774100.795310:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774100.795312:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774101.819249:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774101.819255:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774101.819257:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774101.819276:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774101.819282:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774101.819286:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774101.819293:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774101.819300:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774101.819304:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774101.819918:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774101.819923:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774101.820476:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774101.820480:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774101.820738:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774101.820740:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000100:00000001:2.0:1775774102.843233:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774102.843238:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774102.843240:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774102.843255:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774102.843258:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774102.843261:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774102.843263:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774102.843264:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774102.843266:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774102.971370:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774102.971376:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774102.971381:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774103.867272:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774103.867280:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774103.867287:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774103.867294:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774103.867298:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774103.867302:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774103.867305:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774103.867307:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774103.867310:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774104.891259:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774104.891267:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774104.891269:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774104.891273:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774104.891276:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774104.891280:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774104.891283:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774104.891285:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774104.891287:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774105.915255:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774105.915261:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774105.915262:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774105.915267:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774105.915270:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774105.915273:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774105.915275:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774105.915277:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774105.915279:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774106.939290:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774106.939297:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774106.939303:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774106.939314:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774106.939335:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774106.939340:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774106.939346:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774106.939351:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774106.939359:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774106.939881:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774106.939886:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774106.940505:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774106.940508:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774106.940727:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774106.940729:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.0:1775774107.771286:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774107.771292:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774107.771297:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000100:00000001:2.0:1775774107.963294:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774107.963302:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774107.963304:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774107.963310:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774107.963314:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774107.963318:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774107.963321:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774107.963324:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774107.963327:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774108.091376:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774108.091380:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774108.091384:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774108.987217:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774108.987222:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774108.987224:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774108.987228:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774108.987231:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774108.987234:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774108.987237:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774108.987238:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774108.987240:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774110.011257:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774110.011262:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774110.011264:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774110.011268:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774110.011271:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774110.011274:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774110.011276:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774110.011278:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774110.011280:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774111.035214:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774111.035220:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774111.035222:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774111.035228:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774111.035232:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774111.035235:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774111.035239:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774111.035241:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774111.035244:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774112.059259:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774112.059266:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774112.059268:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774112.059274:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774112.059278:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774112.059283:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774112.059288:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774112.059292:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774112.059297:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774112.059886:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774112.059890:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774112.060401:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774112.060404:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774112.060716:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774112.060719:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000100:00000001:2.0:1775774113.083237:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774113.083242:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774113.083243:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774113.083247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774113.083249:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774113.083252:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774113.083254:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774113.083255:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774113.083257:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774113.211320:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774113.211323:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774113.211326:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774114.107232:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774114.107237:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774114.107238:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774114.107242:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774114.107245:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774114.107248:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774114.107250:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774114.107251:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774114.107253:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774115.131207:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774115.131212:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774115.131214:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774115.131218:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774115.131220:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774115.131223:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774115.131232:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774115.131234:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774115.131236:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774116.155232:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774116.155239:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774116.155241:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774116.155246:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774116.155250:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774116.155254:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774116.155256:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774116.155259:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774116.155261:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774117.179207:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774117.179212:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774117.179214:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774117.179219:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774117.179222:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774117.179225:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774117.179229:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774117.179232:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774117.179235:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774117.179703:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774117.179876:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774117.180037:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:2.0:1775774118.203239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774118.203244:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774118.203246:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774118.203251:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774118.203254:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774118.203257:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774118.203259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774118.203262:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774118.203265:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774118.331314:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774118.331318:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774118.331321:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774119.227210:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774119.227215:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774119.227217:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774119.227220:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774119.227224:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774119.227227:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774119.227228:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774119.227230:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774119.227232:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774120.251254:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774120.251260:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774120.251262:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774120.251267:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774120.251276:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774120.251280:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774120.251283:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774120.251285:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774120.251288:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774121.275230:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774121.275238:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774121.275240:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774121.275248:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774121.275253:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774121.275257:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774121.275260:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774121.275262:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774121.275265:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774122.299271:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774122.299279:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774122.299281:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774122.299286:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774122.299289:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774122.299294:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774122.299297:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774122.299300:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774122.299302:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774122.299856:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774122.299861:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 57 secs later) 00000800:00000200:1.1:1775774122.300347:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774122.300349:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 57 secs later) 00000800:00000200:1.1:1775774122.300596:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774122.300599:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 57 secs later) 00000100:00000001:2.0:1775774123.323214:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774123.323219:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774123.323221:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774123.323224:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774123.323227:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774123.323229:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774123.323232:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774123.323233:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774123.323235:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774123.451314:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774123.451324:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774123.451330:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774124.347222:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774124.347236:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774124.347238:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774124.347244:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774124.347247:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774124.347250:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774124.347253:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774124.347255:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774124.347258:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774125.371234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774125.371238:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774125.371240:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774125.371243:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774125.371246:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774125.371248:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774125.371250:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774125.371251:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774125.371253:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774126.395207:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774126.395214:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774126.395216:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774126.395221:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774126.395224:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774126.395228:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774126.395230:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774126.395232:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774126.395235:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774127.419238:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774127.419245:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774127.419249:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774127.419253:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774127.419258:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774127.419263:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774127.419267:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774127.419269:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774127.419271:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775774127.419662:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774127.419665:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775774127.420015:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774127.420017:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1775774127.420215:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774127.420217:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000100:00000001:2.0:1775774128.443185:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774128.443191:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774128.443192:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774128.443197:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774128.443199:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774128.443202:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774128.443204:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774128.443206:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774128.443207:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774128.571328:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774128.571332:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774128.571334:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774129.467242:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774129.467248:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774129.467250:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774129.467255:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774129.467259:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774129.467263:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774129.467265:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774129.467267:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774129.467270:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774130.491187:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774130.491193:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774130.491195:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774130.491201:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774130.491205:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774130.491208:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774130.491210:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774130.491213:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774130.491215:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774131.515226:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774131.515230:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774131.515232:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774131.515236:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774131.515238:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774131.515241:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774131.515243:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774131.515244:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774131.515246:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774132.539191:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774132.539196:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774132.539198:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774132.539205:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774132.539209:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774132.539213:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774132.539215:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774132.539216:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774132.539218:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775774132.539794:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774132.539800:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775774132.540090:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774132.540092:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1775774132.540351:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774132.540353:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000100:00000001:2.0:1775774133.563259:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774133.563265:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774133.563267:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774133.563273:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774133.563276:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774133.563279:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774133.563282:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774133.563284:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774133.563286:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774133.691318:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774133.691322:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774133.691326:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774134.587238:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774134.587242:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774134.587244:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774134.587248:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774134.587251:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774134.587254:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774134.587256:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774134.587258:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774134.587259:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774135.611234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774135.611240:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774135.611242:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774135.611248:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774135.611252:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774135.611255:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774135.611258:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774135.611260:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774135.611264:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774136.635210:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774136.635216:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774136.635217:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774136.635221:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774136.635223:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774136.635227:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774136.635229:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774136.635232:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774136.635235:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774137.659238:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774137.659244:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774137.659248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774137.659253:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774137.659259:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774137.659263:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774137.659268:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774137.659271:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774137.659277:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775774137.659782:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774137.659788:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775774137.660238:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774137.660240:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1775774137.660512:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774137.660515:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000100:00000001:2.0:1775774138.683254:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774138.683260:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774138.683263:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774138.683268:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774138.683271:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774138.683275:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774138.683278:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774138.683288:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774138.683294:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774138.811372:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774138.811377:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774138.811381:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774139.707264:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774139.707271:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774139.707274:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774139.707279:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774139.707282:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774139.707285:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774139.707288:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774139.707290:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774139.707292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774140.731286:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774140.731295:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774140.731297:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774140.731305:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774140.731309:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774140.731314:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774140.731318:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774140.731320:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774140.731323:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774141.755234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774141.755239:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774141.755241:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774141.755246:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774141.755250:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774141.755254:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774141.755256:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774141.755258:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774141.755260:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774142.779239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774142.779246:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774142.779252:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774142.779260:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774142.779268:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774142.779272:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774142.779274:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774142.779279:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774142.779284:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774142.779739:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774142.779743:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775774142.780189:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774142.780192:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775774142.780459:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774142.780461:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000100:00000001:2.0:1775774143.803251:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774143.803259:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774143.803260:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774143.803266:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774143.803269:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774143.803274:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774143.803276:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774143.803278:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774143.803281:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774143.931341:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774143.931346:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774143.931350:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774144.827280:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774144.827286:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774144.827288:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774144.827293:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774144.827297:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774144.827301:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774144.827304:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774144.827306:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774144.827308:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774145.851241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774145.851250:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774145.851251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774145.851258:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774145.851262:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774145.851266:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774145.851269:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774145.851272:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774145.851274:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775774146.683226:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774146.683235:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774146.683242:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000100:00000001:2.0:1775774146.875259:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774146.875266:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774146.875268:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774146.875273:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774146.875277:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774146.875281:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774146.875285:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774146.875287:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774146.875292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774147.899194:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774147.899200:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774147.899202:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774147.899208:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774147.899212:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774147.899216:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774147.899219:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774147.899221:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774147.899230:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774147.899714:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774147.899721:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775774147.900091:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774147.900093:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775774147.900398:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774147.900401:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000100:00000001:2.0:1775774148.923260:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774148.923266:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774148.923268:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774148.923274:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774148.923277:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774148.923282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774148.923284:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774148.923287:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774148.923289:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774149.051319:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774149.051323:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774149.051326:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774149.947207:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774149.947216:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774149.947218:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774149.947225:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774149.947230:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774149.947234:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774149.947237:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774149.947240:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774149.947244:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774150.971262:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774150.971270:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774150.971271:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774150.971277:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774150.971280:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774150.971284:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774150.971287:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774150.971290:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774150.971293:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774151.995219:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774151.995223:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774151.995224:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774151.995228:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774151.995231:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774151.995233:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774151.995235:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774151.995237:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774151.995238:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774153.019210:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774153.019216:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774153.019217:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774153.019221:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774153.019225:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774153.019228:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774153.019230:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774153.019231:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774153.019233:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774153.019769:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774153.019773:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775774153.020121:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774153.020123:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775774153.020397:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774153.020400:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000100:00000001:2.0:1775774154.043258:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774154.043264:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774154.043266:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774154.043271:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774154.043275:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774154.043278:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774154.043281:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774154.043283:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774154.043285:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774154.171292:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774154.171295:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774154.171298:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774155.067215:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774155.067219:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774155.067220:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774155.067224:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774155.067226:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774155.067229:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774155.067232:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774155.067233:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774155.067235:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774156.091240:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774156.091246:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774156.091248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774156.091252:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774156.091255:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774156.091258:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774156.091261:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774156.091262:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774156.091264:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774157.115227:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774157.115234:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774157.115236:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774157.115242:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774157.115246:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774157.115250:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774157.115253:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774157.115256:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774157.115258:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774158.139228:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774158.139234:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774158.139238:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774158.139244:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774158.139249:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774158.139254:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774158.139260:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774158.139263:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774158.139269:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774158.139751:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774158.139757:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774158.140129:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774158.140130:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774158.140376:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774158.140378:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1775774159.163239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774159.163246:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774159.163248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774159.163268:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774159.163272:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774159.163275:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774159.163277:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774159.163279:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774159.163282:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774159.291253:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774159.291274:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774159.291277:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774160.187229:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774160.187235:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774160.187236:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774160.187251:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774160.187255:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774160.187258:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774160.187259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774160.187261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774160.187264:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774161.211215:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774161.211221:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774161.211235:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774161.211240:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774161.211243:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774161.211245:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774161.211247:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774161.211249:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774161.211251:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774162.235270:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774162.235277:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774162.235279:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774162.235285:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774162.235289:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774162.235293:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774162.235295:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774162.235298:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774162.235300:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774163.259245:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774163.259254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774163.259256:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774163.259263:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774163.259267:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774163.259271:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774163.259274:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774163.259278:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774163.259281:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774163.259878:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774163.259884:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774163.260423:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774163.260426:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774163.260734:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774163.260737:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1775774164.283272:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774164.283277:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774164.283279:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774164.283284:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774164.283286:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774164.283289:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774164.283291:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774164.283293:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774164.283295:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774164.411349:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774164.411352:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774164.411356:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774165.307204:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774165.307210:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774165.307211:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774165.307216:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774165.307219:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774165.307222:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774165.307224:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774165.307226:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774165.307228:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774166.331247:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774166.331253:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774166.331254:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774166.331259:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774166.331262:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774166.331265:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774166.331267:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774166.331269:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774166.331271:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774167.355205:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774167.355211:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774167.355212:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774167.355217:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774167.355220:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774167.355224:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774167.355228:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774167.355230:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774167.355233:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774168.379229:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774168.379236:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774168.379238:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774168.379243:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774168.379247:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774168.379253:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774168.379257:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774168.379261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774168.379266:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774168.379778:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774168.379781:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1775774168.380081:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774168.380083:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1775774168.380437:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774168.380440:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 11 secs later) 00000100:00000001:2.0:1775774169.403188:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774169.403195:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774169.403198:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774169.403204:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774169.403208:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774169.403211:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774169.403214:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774169.403217:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774169.403219:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774169.531311:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774169.531315:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774169.531319:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774170.427243:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774170.427250:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774170.427252:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774170.427258:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774170.427262:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774170.427265:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774170.427268:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774170.427270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774170.427273:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774171.451224:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774171.451230:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774171.451233:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774171.451237:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774171.451240:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774171.451244:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774171.451246:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774171.451248:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774171.451250:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774173.499728:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774173.499735:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775774173.500198:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774173.500200:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775774173.500428:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774173.500431:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000020:00000001:2.0:1775774174.651304:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774174.651307:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774174.651310:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774176.571274:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774176.571280:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774176.571282:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774176.571287:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774176.571290:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774176.571293:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774176.571295:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774176.571297:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774176.571299:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774177.595200:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774177.595206:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774177.595208:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774177.595213:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774177.595216:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774177.595218:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774177.595220:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774177.595221:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774177.595223:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774178.619231:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774178.619238:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774178.619240:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774178.619245:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774178.619249:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774178.619252:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774178.619255:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774178.619257:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774178.619260:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774178.619768:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774178.619773:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775774178.620122:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774178.620124:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775774178.620357:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774178.620360:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000100:00000001:2.0:1775774179.643204:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774179.643210:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774179.643212:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774179.643216:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774179.643220:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774179.643223:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774179.643225:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774179.643227:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774179.643229:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774179.771304:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774179.771308:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774179.771312:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774180.667216:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774180.667223:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774180.667224:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774180.667229:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774180.667231:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774180.667234:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774180.667236:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774180.667238:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774180.667240:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774181.691216:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774181.691221:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774181.691223:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774181.691230:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774181.691234:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774181.691237:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774181.691240:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774181.691243:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774181.691246:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774182.715203:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774182.715210:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774182.715212:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774182.715218:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774182.715222:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774182.715226:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774182.715228:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774182.715231:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774182.715234:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774183.739237:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774183.739243:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774183.739247:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774183.739254:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774183.739259:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774183.739264:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774183.739268:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774183.739273:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774183.739278:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774183.739794:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774183.740051:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774183.740276:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:2.0:1775774184.571267:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774184.571273:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774184.571277:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000020:00000001:2.0:1775774184.891302:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774184.891306:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774184.891310:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774186.811292:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774186.811300:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774186.811302:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774186.811308:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774186.811312:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774186.811316:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774186.811318:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774186.811321:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774186.811323:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774187.835287:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774187.835295:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774187.835297:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774187.835303:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774187.835307:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774187.835311:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774187.835313:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774187.835316:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774187.835318:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774188.859214:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774188.859223:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774188.859224:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774188.859229:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774188.859234:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774188.859238:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774188.859240:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774188.859244:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774188.859247:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774188.859285:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774188.859289:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774188.859292:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774188.859295:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774188.859299:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774188.859306:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11071 00000400:00000200:2.0:1775774188.859319:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774188.859325:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774188.859327:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774188.859330:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774188.859333:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774188.859335:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774188.859337:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774188.859341:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774188.859342:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774188.859345:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774188.859348:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774188.859349:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774188.859351:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774188.859352:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774188.859354:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774188.859355:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774188.859356:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774188.859359:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774188.859362:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774188.859364:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774188.859367:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774188.859370:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774188.859372:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774188.859375:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774188.859377:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774188.859384:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775774188.859388:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774188.859392:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774188.859393:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774188.859396:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774188.859399:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774188.859404:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774188.859412:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775774188.859419:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774188.859428:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710245632/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774188.859438:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774188.859443:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774188.859446:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710245632/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774188.859469:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774188.859471:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774188.859474:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774188.859475:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774188.859476:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774188.859479:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774188.859480:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11071 00000400:00000200:2.0:1775774188.859483:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774188.859485:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774188.859487:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774188.859489:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774188.859491:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774188.859492:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774188.859495:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774188.859497:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774188.859498:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774188.859499:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774188.859501:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774188.859503:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774188.859505:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774188.859507:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774188.859508:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774188.859509:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774188.859511:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774188.859513:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774188.859517:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774188.859520:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774188.859522:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774188.859530:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774188.859534:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774188.859539:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774188.859546:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774188.859564:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774188.859581:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774188.859595:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775774188.859602:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774188.859609:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774188.859612:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774188.859619:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774188.859622:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774188.859626:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774188.859634:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775774188.859642:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774188.859655:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710245760/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774188.859670:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774188.859676:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774188.859686:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710245760/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774188.859717:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774188.859721:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774188.859730:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774188.859732:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774188.859738:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774188.859760:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774188.859763:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11071 00000400:00000200:2.0:1775774188.859771:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774188.859781:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774188.859788:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774188.859796:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774188.859805:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774188.859810:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774188.859819:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774188.859825:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774188.859829:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774188.859834:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774188.859839:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774188.859844:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774188.859850:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774188.859860:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774188.859865:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774188.859873:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774188.859879:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774188.859885:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774188.859887:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774188.859893:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774188.859900:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774188.859908:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774188.859911:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774188.859914:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774188.859917:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774188.859930:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774188.859938:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774188.859945:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775774188.859954:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774188.859960:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774188.859963:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774188.859969:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774188.859977:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774188.859984:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774188.859993:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775774188.860005:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774188.860020:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710245888/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774188.860051:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774188.860055:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774188.860069:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710245888/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774188.860101:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774188.860111:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774188.860118:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774188.860127:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775774188.860175:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774188.860179:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000100:00000040:2.0:1775774188.860195:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774188.860206:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11081) 00000800:00000200:1.1:1775774188.860609:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774188.860612:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774188.860920:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774188.860922:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000100:00000001:2.0:1775774189.883234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774189.883240:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774189.883241:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774189.883246:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774189.883249:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774189.883252:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774189.883254:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774189.883256:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774189.883258:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774190.011359:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774190.011363:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774190.011367:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774190.907212:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774190.907219:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774190.907221:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774190.907226:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774190.907230:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774190.907234:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774190.907237:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774190.907239:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774190.907242:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774191.931242:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774191.931247:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774191.931249:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774191.931253:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774191.931256:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774191.931260:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774191.931262:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774191.931264:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774191.931266:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774192.955216:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774192.955221:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774192.955223:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774192.955228:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774192.955231:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774192.955235:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774192.955237:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774192.955238:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774192.955241:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774193.979241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774193.979248:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774193.979251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774193.979257:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774193.979262:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774193.979265:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774193.979269:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774193.979271:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774193.979275:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774193.979770:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774193.979775:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774193.980055:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774193.980057:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774193.980262:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774193.980264:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000100:00000001:2.0:1775774195.003204:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774195.003210:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774195.003211:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774195.003216:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774195.003219:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774195.003222:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774195.003224:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774195.003226:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774195.003234:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774195.131314:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774195.131326:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774195.131334:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774196.027310:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774196.027314:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774196.027316:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774196.027321:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774196.027324:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774196.027327:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774196.027329:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774196.027331:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774196.027332:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774197.051224:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774197.051231:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774197.051234:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774197.051239:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774197.051243:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774197.051247:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774197.051250:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774197.051253:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774197.051257:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774198.075226:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774198.075231:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774198.075233:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774198.075237:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774198.075240:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774198.075243:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774198.075245:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774198.075246:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774198.075248:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774199.099211:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774199.099219:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774199.099222:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774199.099225:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774199.099231:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774199.099239:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11081 00000400:00000200:2.0:1775774199.099250:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774199.099262:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774199.099265:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774199.099268:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774199.099271:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774199.099273:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774199.099275:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774199.099278:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774199.099279:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774199.099281:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774199.099283:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774199.099285:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774199.099286:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774199.099289:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774199.099290:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774199.099291:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774199.099294:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774199.099296:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774199.099298:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774199.099300:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774199.099303:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774199.099305:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774199.099307:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774199.099309:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774199.099310:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774199.099317:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775774199.099320:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774199.099323:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774199.099324:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774199.099326:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774199.099328:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774199.099331:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774199.099337:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775774199.099343:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774199.099352:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710246400/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774199.099369:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774199.099375:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774199.099380:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710246400/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774199.099402:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774199.099406:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774199.099408:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774199.099409:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774199.099411:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774199.099413:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774199.099415:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11081 00000400:00000200:2.0:1775774199.099418:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774199.099421:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774199.099423:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774199.099425:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774199.099427:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774199.099429:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774199.099432:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774199.099435:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774199.099436:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774199.099438:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774199.099441:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774199.099446:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774199.099451:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774199.099469:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774199.099473:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774199.099478:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774199.099482:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774199.099486:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774199.099493:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774199.099496:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774199.099498:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774199.099504:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774199.099507:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774199.099513:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774199.099518:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774199.099524:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774199.099527:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774199.099531:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775774199.099534:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774199.099536:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774199.099538:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774199.099540:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774199.099541:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774199.099543:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774199.099550:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775774199.099556:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774199.099584:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710246528/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774199.099598:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774199.099602:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774199.099609:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710246528/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774199.099623:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774199.099627:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774199.099631:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774199.099634:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774199.099637:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774199.099640:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774199.099644:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11081 00000400:00000200:2.0:1775774199.099653:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774199.099659:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774199.099664:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774199.099676:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774199.099681:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774199.099686:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774199.099691:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774199.099695:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774199.099696:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774199.099698:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774199.099701:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774199.099703:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774199.099707:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774199.099712:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774199.099715:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774199.099719:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774199.099721:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774199.099724:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000800:00000200:1.1:1775774199.099725:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00080000:2.0:1775774199.099729:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000800:00000200:1.1:1775774199.099730:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1775774199.099734:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774199.099737:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774199.099740:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774199.099744:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774199.099747:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774199.099749:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774199.099753:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774199.099758:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774199.099765:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775774199.099769:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774199.099774:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774199.099777:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774199.099781:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774199.099784:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774199.099788:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774199.099795:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775774199.099802:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774199.099813:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710246656/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774199.099826:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774199.099829:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774199.099836:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710246656/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774199.099848:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774199.099852:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774199.099863:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774199.099867:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774199.099871:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774199.099878:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11091) 00000800:00000200:1.1:1775774199.100086:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774199.100087:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774199.100300:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774199.100301:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000020:00000001:2.0:1775774200.251279:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774200.251282:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774200.251285:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774204.219770:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774204.219775:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774204.220125:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774204.220127:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774204.220399:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774204.220402:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000020:00000001:2.0:1775774205.371300:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774205.371304:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774205.371306:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774209.339964:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774209.339971:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775774209.340531:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774209.340534:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775774209.340853:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774209.340855:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000020:00000001:2.0:1775774210.491413:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774210.491419:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774210.491424:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774214.459760:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774214.459766:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775774214.460180:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774214.460182:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775774214.460420:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774214.460421:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000020:00000001:2.0:1775774215.611354:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774215.611358:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774215.611360:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774219.579821:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774219.579824:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775774219.580308:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774219.580311:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775774219.580617:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774219.580620:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000020:00000001:2.0:1775774220.731313:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774220.731317:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774220.731320:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.0:1775774223.483241:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774223.483251:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774223.483257:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:1.1:1775774224.699836:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774224.699840:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774224.700403:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774224.700406:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774224.700689:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774224.700690:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000020:00000001:2.0:1775774225.851339:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774225.851342:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774225.851345:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774229.819798:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774229.819806:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774229.820248:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774229.820251:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774229.820486:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774229.820488:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1775774230.971307:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774230.971310:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774230.971313:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774234.939783:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774234.939787:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774234.940199:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774234.940201:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774234.940438:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774234.940440:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000020:00000001:2.0:1775774236.091302:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774236.091305:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774236.091308:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774240.059694:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774240.059701:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774240.060090:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774240.060091:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774240.060361:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774240.060363:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000020:00000001:2.0:1775774241.211289:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774241.211294:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774241.211297:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774245.179260:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774245.179271:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774245.179277:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774245.179282:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774245.179293:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774245.179305:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11127 00000400:00000200:2.0:1775774245.179319:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774245.179327:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774245.179331:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774245.179336:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774245.179340:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774245.179343:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774245.179345:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774245.179350:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774245.179352:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774245.179355:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774245.179357:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774245.179360:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774245.179362:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774245.179365:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774245.179367:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774245.179369:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774245.179371:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774245.179374:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774245.179377:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774245.179381:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774245.179384:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774245.179388:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774245.179391:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774245.179394:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774245.179396:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774245.179405:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775774245.179409:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774245.179414:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774245.179415:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774245.179418:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774245.179421:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774245.179425:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774245.179433:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775774245.179439:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774245.179448:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710249856/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774245.179460:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774245.179464:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774245.179469:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710249856/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774245.179492:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774245.179495:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774245.179499:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774245.179500:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774245.179502:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774245.179504:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774245.179507:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11127 00000400:00000200:2.0:1775774245.179511:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774245.179514:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774245.179516:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774245.179520:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774245.179522:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774245.179526:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774245.179529:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774245.179532:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774245.179533:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774245.179535:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774245.179539:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774245.179545:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774245.179549:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774245.179556:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774245.179562:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774245.179567:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774245.179573:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774245.179577:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774245.179583:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774245.179588:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774245.179592:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774245.179600:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774245.179605:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774245.179611:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774245.179617:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774245.179623:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774245.179627:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774245.179634:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775774245.179637:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774245.179641:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774245.179644:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774245.179646:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774245.179649:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774245.179652:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774245.179660:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775774245.179667:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774245.179676:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710249984/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774245.179696:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774245.179701:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774245.179711:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710249984/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774245.179731:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774245.179736:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774245.179744:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774245.179749:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774245.179753:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774245.179772:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774245.179780:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11127 00000400:00000200:2.0:1775774245.179791:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774245.179800:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774245.179807:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774245.179816:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774245.179821:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774245.179826:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774245.179831:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774245.179836:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774245.179840:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774245.179844:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774245.179848:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000800:00000200:1.1:1775774245.179848:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774245.179852:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000020:00000001:2.0:1775774245.179853:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774245.179859:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774245.179868:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774245.179873:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774245.179880:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774245.179885:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774245.179891:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774245.179896:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774245.179903:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774245.179908:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774245.179914:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774245.179920:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774245.179926:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774245.179931:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774245.179937:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774245.179940:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774245.179946:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775774245.179952:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774245.179956:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774245.179959:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774245.179962:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774245.179965:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774245.179967:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774245.179973:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775774245.179977:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774245.179987:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710250112/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774245.180006:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774245.180011:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774245.180022:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710250112/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774245.180043:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774245.180049:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774245.180056:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774245.180061:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774245.180070:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774245.180077:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11137) 00000800:00000200:1.1:1775774245.180311:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774245.180313:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774245.180503:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774245.180505:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000020:00000001:2.0:1775774246.331309:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774246.331312:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774246.331321:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774250.299312:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774250.299322:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774250.299328:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774250.299336:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774250.299344:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774250.299348:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774250.299355:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774250.299359:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774250.299362:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774250.299922:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774250.300204:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774250.300360:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:2.0:1775774251.323253:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774251.323258:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774251.323260:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774251.323265:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774251.323268:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774251.323271:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774251.323274:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774251.323275:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774251.323277:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774251.451401:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774251.451406:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774251.451411:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774252.347224:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774252.347231:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774252.347233:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774252.347241:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774252.347246:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774252.347251:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774252.347255:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774252.347258:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774252.347261:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774253.371248:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774253.371256:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774253.371258:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774253.371265:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774253.371268:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774253.371272:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774253.371274:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774253.371277:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774253.371280:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774254.395344:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774254.395352:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774254.395354:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774254.395360:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774254.395364:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774254.395368:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774254.395371:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774254.395373:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774254.395375:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774255.419423:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774255.419432:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774255.419437:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774255.419450:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774255.419461:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774255.419467:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774255.419471:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774255.419474:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774255.419478:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774255.419927:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774255.419934:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774255.420461:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774255.420464:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774255.420792:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774255.420794:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000100:00000001:2.0:1775774256.443421:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774256.443431:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774256.443433:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774256.443441:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774256.443445:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774256.443450:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774256.443454:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774256.443457:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774256.443461:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774256.571420:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774256.571424:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774256.571429:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774257.467268:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774257.467276:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774257.467278:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774257.467284:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774257.467287:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774257.467292:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774257.467294:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774257.467297:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774257.467300:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774258.491310:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774258.491321:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774258.491324:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774258.491338:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774258.491343:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774258.491348:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774258.491352:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774258.491355:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774258.491358:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774259.515177:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774259.515183:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774259.515185:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774259.515190:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774259.515193:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774259.515195:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774259.515198:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774259.515200:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774259.515202:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774260.539202:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774260.539208:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774260.539210:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774260.539215:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774260.539218:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774260.539221:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774260.539223:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774260.539225:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774260.539228:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774260.539759:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774260.539763:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774260.540092:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774260.540094:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774260.540314:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774260.540320:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.0:1775774261.371278:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774261.371286:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774261.371292:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000100:00000001:2.0:1775774261.563241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774261.563247:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774261.563249:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774261.563253:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774261.563256:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774261.563260:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774261.563262:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774261.563264:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774261.563266:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774261.691403:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774261.691408:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774261.691411:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774262.587210:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774262.587218:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774262.587220:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774262.587227:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774262.587231:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774262.587235:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774262.587238:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774262.587241:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774262.587244:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774263.611226:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774263.611231:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774263.611232:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774263.611238:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774263.611240:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774263.611243:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774263.611246:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774263.611248:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774263.611250:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774264.635199:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774264.635206:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774264.635209:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774264.635215:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774264.635218:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774264.635222:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774264.635225:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774264.635228:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774264.635231:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774265.659258:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774265.659264:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774265.659266:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774265.659273:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774265.659279:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774265.659285:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774265.659290:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774265.659295:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774265.659300:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774265.659818:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774265.659822:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774265.660266:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774265.660268:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774265.660559:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774265.660560:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1775774266.683218:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774266.683225:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774266.683227:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774266.683234:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774266.683238:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774266.683242:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774266.683244:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774266.683247:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774266.683249:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774266.811292:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774266.811296:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774266.811300:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774267.707234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774267.707241:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774267.707243:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774267.707250:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774267.707254:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774267.707258:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774267.707261:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774267.707263:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774267.707266:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774268.731248:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774268.731254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774268.731255:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774268.731260:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774268.731263:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774268.731266:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774268.731268:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774268.731270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774268.731272:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774269.755258:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774269.755265:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774269.755268:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774269.755274:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774269.755278:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774269.755282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774269.755285:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774269.755287:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774269.755290:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774270.779238:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774270.779246:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774270.779249:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774270.779254:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774270.779258:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774270.779262:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774270.779267:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774270.779270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774270.779273:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775774270.779806:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774270.779811:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775774270.780273:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774270.780275:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1775774270.780486:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774270.780488:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000100:00000001:2.0:1775774271.803217:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774271.803224:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774271.803226:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774271.803232:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774271.803235:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774271.803240:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774271.803242:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774271.803245:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774271.803247:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774271.931337:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774271.931341:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774271.931344:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774272.827260:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774272.827267:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774272.827269:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774272.827307:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774272.827312:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774272.827315:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774272.827319:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774272.827322:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774272.827325:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774273.851214:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774273.851219:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774273.851220:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774273.851224:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774273.851227:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774273.851235:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774273.851237:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774273.851238:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774273.851240:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774274.875212:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774274.875218:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774274.875220:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774274.875225:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774274.875229:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774274.875232:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774274.875235:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774274.875237:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774274.875252:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774275.899238:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774275.899243:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774275.899245:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774275.899250:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774275.899253:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774275.899256:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774275.899258:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774275.899260:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774275.899262:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774275.899728:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774275.899735:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775774275.900260:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774275.900263:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775774275.900536:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774275.900538:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000100:00000001:2.0:1775774276.923195:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774276.923200:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774276.923201:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774276.923223:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774276.923226:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774276.923229:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774276.923231:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774276.923232:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774276.923235:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774277.051328:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774277.051331:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774277.051333:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774277.947209:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774277.947214:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774277.947216:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774277.947220:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774277.947222:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774277.947224:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774277.947226:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774277.947227:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774277.947229:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774278.971228:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774278.971235:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774278.971237:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774278.971247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774278.971251:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774278.971254:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774278.971257:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774278.971259:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774278.971262:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774279.995256:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774279.995262:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774279.995264:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774279.995269:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774279.995273:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774279.995276:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774279.995278:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774279.995280:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774279.995282:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774281.019242:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774281.019248:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774281.019250:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774281.019255:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774281.019259:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774281.019262:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774281.019263:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774281.019266:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774281.019267:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774281.019921:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774281.019925:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775774281.020390:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774281.020392:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775774281.020679:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774281.020681:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000100:00000001:2.0:1775774282.043239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774282.043247:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774282.043250:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774282.043256:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774282.043261:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774282.043265:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774282.043268:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774282.043271:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774282.043274:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774282.171303:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774282.171309:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774282.171314:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774283.067262:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774283.067270:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774283.067272:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774283.067277:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774283.067281:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774283.067285:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774283.067288:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774283.067290:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774283.067292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774284.091255:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774284.091262:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774284.091264:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774284.091269:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774284.091273:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774284.091276:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774284.091279:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774284.091281:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774284.091284:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774285.115290:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774285.115298:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774285.115301:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774285.115307:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774285.115312:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774285.115317:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774285.115322:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774285.115325:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774285.115328:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774286.139208:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774286.139220:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774286.139223:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774286.139231:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774286.139237:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774286.139242:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774286.139245:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774286.139248:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774286.139252:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774286.139294:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774286.139299:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774286.139302:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774286.139305:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774286.139308:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774286.139315:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11168 00000400:00000200:2.0:1775774286.139325:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774286.139331:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774286.139334:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774286.139337:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774286.139339:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774286.139342:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774286.139343:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774286.139347:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774286.139348:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774286.139350:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774286.139353:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774286.139355:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774286.139356:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774286.139358:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774286.139360:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774286.139362:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774286.139363:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774286.139366:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774286.139369:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774286.139370:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774286.139373:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774286.139376:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774286.139379:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774286.139381:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774286.139384:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774286.139393:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775774286.139397:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774286.139402:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774286.139404:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774286.139407:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774286.139410:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774286.139415:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774286.139425:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775774286.139434:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774286.139446:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710252928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774286.139459:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774286.139465:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774286.139471:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710252928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774286.139499:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774286.139503:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774286.139506:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774286.139508:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774286.139511:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774286.139514:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774286.139519:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11168 00000400:00000200:2.0:1775774286.139525:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774286.139531:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774286.139535:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774286.139541:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774286.139548:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774286.139551:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774286.139553:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774286.139557:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774286.139561:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774286.139566:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774286.139568:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774286.139571:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774286.139576:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774286.139584:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774286.139594:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774286.139600:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774286.139609:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774286.139614:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774286.139620:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774286.139627:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774286.139631:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774286.139641:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774286.139644:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774286.139648:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774286.139653:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774286.139658:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774286.139660:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774286.139669:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775774286.139674:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774286.139681:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774286.139684:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774286.139686:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774286.139687:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774286.139690:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774286.139695:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775774286.139699:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774286.139706:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710253056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774286.139725:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774286.139727:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774286.139731:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710253056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774286.139752:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774286.139757:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774286.139761:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774286.139766:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774286.139767:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774286.139770:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774286.139777:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11168 00000400:00000200:2.0:1775774286.139787:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774286.139795:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774286.139798:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774286.139803:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774286.139806:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774286.139810:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774286.139825:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774286.139832:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774286.139837:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774286.139843:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774286.139845:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774286.139853:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774286.139857:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774286.139866:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774286.139875:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774286.139880:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774286.139882:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774286.139884:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774286.139888:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774286.139894:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774286.139900:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774286.139903:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774286.139904:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774286.139906:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774286.139907:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774286.139912:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774286.139918:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774286.139925:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775774286.139929:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774286.139935:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774286.139937:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774286.139939:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774286.139943:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774286.139948:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774286.139953:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775774286.139960:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774286.139969:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710253184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774286.139990:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774286.139992:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774286.140006:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710253184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774286.140033:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774286.140036:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774286.140058:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774286.140061:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774286.140064:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774286.140071:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11178) 00000800:00000200:1.1:1775774286.140092:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774286.140095:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775774286.140406:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774286.140407:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775774286.140637:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774286.140639:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000100:00000001:2.0:1775774287.163276:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774287.163284:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774287.163287:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774287.163295:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774287.163300:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774287.163305:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774287.163308:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774287.163310:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774287.163312:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774287.292058:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774287.292064:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774287.292067:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774288.187219:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774288.187225:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774288.187226:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774288.187230:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774288.187233:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774288.187235:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774288.187237:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774288.187239:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774288.187240:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774289.211235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774289.211239:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774289.211240:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774289.211244:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774289.211247:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774289.211250:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774289.211251:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774289.211253:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774289.211256:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774290.235266:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774290.235272:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774290.235275:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774290.235280:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774290.235284:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774290.235287:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774290.235290:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774290.235292:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774290.235294:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774291.259246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774291.259251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774291.259253:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774291.259257:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774291.259261:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774291.259264:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774291.259267:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774291.259271:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774291.259275:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774291.259748:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774291.259752:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 23 secs later) 00000800:00000200:1.1:1775774291.260126:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774291.260128:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 23 secs later) 00000800:00000200:1.1:1775774291.260368:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774291.260369:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 23 secs later) 00000100:00000001:2.0:1775774292.283233:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774292.283239:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774292.283241:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774292.283247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774292.283250:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774292.283255:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774292.283257:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774292.283259:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774292.283262:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774292.411379:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774292.411384:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774292.411388:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774293.307270:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774293.307277:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774293.307279:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774293.307285:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774293.307289:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774293.307293:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774293.307296:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774293.307299:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774293.307301:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774294.331242:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774294.331249:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774294.331250:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774294.331256:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774294.331259:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774294.331263:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774294.331270:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774294.331272:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774294.331274:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775774296.379596:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774296.379600:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775774296.379910:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774296.379911:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1775774296.380071:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774296.380073:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1775774297.531285:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774297.531288:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774297.531291:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.0:1775774300.283245:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774300.283255:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774300.283261:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000100:00080000:2.0:1775774301.499239:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774301.499247:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774301.499251:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774301.499256:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774301.499262:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774301.499271:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11184 00000400:00000200:2.0:1775774301.499280:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774301.499286:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774301.499289:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774301.499292:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774301.499295:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774301.499298:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774301.499300:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774301.499303:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774301.499305:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774301.499307:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774301.499310:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774301.499312:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774301.499314:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774301.499317:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774301.499319:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774301.499320:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774301.499322:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774301.499325:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774301.499327:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774301.499330:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774301.499333:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774301.499337:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774301.499339:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774301.499341:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774301.499343:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774301.499351:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775774301.499355:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774301.499358:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774301.499360:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774301.499362:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774301.499365:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774301.499369:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774301.499376:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775774301.499381:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774301.499389:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710254080/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774301.499398:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774301.499403:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774301.499407:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710254080/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774301.499431:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774301.499433:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774301.499437:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774301.499438:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774301.499440:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774301.499442:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774301.499444:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11184 00000400:00000200:2.0:1775774301.499448:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774301.499451:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774301.499454:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774301.499457:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774301.499459:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774301.499461:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774301.499463:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774301.499466:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774301.499469:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774301.499472:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774301.499477:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774301.499481:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774301.499486:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774301.499495:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774301.499498:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774301.499503:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774301.499507:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774301.499511:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774301.499518:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774301.499522:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774301.499525:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774301.499531:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774301.499535:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774301.499537:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774301.499543:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774301.499549:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774301.499553:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774301.499557:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775774301.499561:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774301.499566:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774301.499568:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774301.499572:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774301.499575:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774301.499578:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774301.499585:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775774301.499591:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774301.499599:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710254208/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774301.499611:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774301.499616:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774301.499625:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710254208/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774301.499647:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774301.499652:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774301.499659:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774301.499663:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774301.499666:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774301.499672:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774301.499677:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11184 00000400:00000200:2.0:1775774301.499685:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000200:3.1:1775774301.499689:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000400:00000200:2.0:1775774301.499692:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000800:00000200:3.1:1775774301.499693:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000100:00080000:2.0:1775774301.499696:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774301.499703:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774301.499706:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774301.499708:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774301.499711:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774301.499713:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774301.499716:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774301.499719:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774301.499722:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774301.499726:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774301.499729:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774301.499734:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774301.499738:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774301.499741:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774301.499744:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774301.499748:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774301.499755:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774301.499760:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774301.499763:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774301.499767:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774301.499770:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774301.499787:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774301.499789:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774301.499792:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774301.499793:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774301.499800:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775774301.499805:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774301.499810:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774301.499813:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774301.499818:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774301.499821:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774301.499824:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774301.499830:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775774301.499834:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774301.499842:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710254336/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774301.499874:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774301.499877:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774301.499887:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710254336/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774301.499917:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774301.499922:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774301.499932:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774301.499938:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774301.499946:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774301.499954:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11194) 00000800:00000200:3.1:1775774301.500152:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774301.500154:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1775774301.500331:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774301.500333:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000020:00000001:2.0:1775774302.651389:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774302.651393:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774302.651397:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774306.619266:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774306.619282:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774306.619289:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774306.619293:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774306.619296:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774306.619304:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11189 00000400:00000200:2.0:1775774306.619314:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774306.619319:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774306.619322:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774306.619324:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774306.619327:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774306.619329:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774306.619330:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774306.619333:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774306.619335:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774306.619338:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774306.619340:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774306.619342:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774306.619343:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774306.619345:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774306.619346:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774306.619347:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774306.619349:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774306.619352:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774306.619355:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774306.619357:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774306.619360:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774306.619369:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774306.619373:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774306.619375:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774306.619377:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774306.619389:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775774306.619393:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774306.619397:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774306.619399:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774306.619402:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774306.619405:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774306.619409:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774306.619417:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775774306.619424:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774306.619433:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710254464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774306.619441:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774306.619446:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774306.619449:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710254464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774306.619474:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774306.619476:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774306.619479:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774306.619480:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774306.619481:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774306.619483:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774306.619485:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11189 00000400:00000200:2.0:1775774306.619488:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774306.619490:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774306.619492:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774306.619494:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774306.619495:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774306.619497:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774306.619498:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774306.619499:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774306.619500:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774306.619502:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774306.619503:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774306.619504:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774306.619505:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774306.619507:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774306.619509:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774306.619510:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774306.619512:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774306.619514:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774306.619516:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774306.619517:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774306.619519:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774306.619524:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774306.619528:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774306.619529:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774306.619534:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774306.619538:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774306.619542:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774306.619548:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775774306.619553:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774306.619560:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774306.619564:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774306.619569:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774306.619572:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774306.619576:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774306.619582:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775774306.619589:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774306.619597:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710254592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774306.619607:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774306.619612:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774306.619618:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710254592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774306.619633:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774306.619636:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774306.619641:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774306.619642:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774306.619643:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774306.619646:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774306.619648:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11189 00000400:00000200:2.0:1775774306.619653:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774306.619658:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774306.619661:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774306.619664:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774306.619666:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774306.619669:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774306.619672:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774306.619675:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774306.619677:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774306.619679:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774306.619681:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774306.619684:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774306.619706:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774306.619711:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774306.619715:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774306.619717:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774306.619719:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774306.619722:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774306.619724:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774306.619728:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774306.619730:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774306.619733:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774306.619734:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774306.619736:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774306.619737:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774306.619741:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774306.619743:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774306.619746:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775774306.619750:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774306.619753:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774306.619755:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774306.619757:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774306.619758:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774306.619761:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774306.619765:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775774306.619770:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774306.619776:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710254720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774306.619788:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774306.619789:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774306.619794:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710254720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774306.619810:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774306.619813:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774306.619817:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774306.619819:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774306.619822:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774306.619826:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11199) 00000800:00000200:3.1:1775774306.619838:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774306.619842:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775774306.620178:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774306.620180:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1775774306.620396:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774306.620397:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000020:00000001:2.0:1775774307.771367:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774307.771372:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774307.771375:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774309.691235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774309.691239:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774309.691241:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774309.691245:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774309.691249:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774309.691251:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774309.691253:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774309.691255:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774309.691257:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774310.715212:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774310.715219:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774310.715221:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774310.715226:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774310.715230:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774310.715234:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774310.715237:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774310.715239:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774310.715242:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774311.739235:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774311.739246:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774311.739249:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774311.739253:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774311.739257:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774311.739268:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11194 00000400:00000200:2.0:1775774311.739280:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774311.739287:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774311.739289:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774311.739292:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774311.739294:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774311.739296:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774311.739298:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774311.739300:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774311.739301:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774311.739303:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774311.739305:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774311.739306:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774311.739307:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774311.739309:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774311.739310:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774311.739311:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774311.739312:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774311.739314:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774311.739316:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774311.739318:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774311.739321:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774311.739324:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774311.739328:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774311.739330:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774311.739333:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774311.739346:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775774311.739350:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774311.739353:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774311.739355:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774311.739358:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774311.739361:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774311.739364:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774311.739371:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775774311.739377:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774311.739386:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710254848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774311.739395:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774311.739399:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774311.739403:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710254848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774311.739417:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774311.739419:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774311.739421:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774311.739422:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774311.739423:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774311.739425:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774311.739427:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11194 00000400:00000200:2.0:1775774311.739431:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774311.739433:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774311.739435:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774311.739437:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774311.739439:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774311.739442:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774311.739446:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774311.739449:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774311.739450:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774311.739453:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774311.739454:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774311.739458:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774311.739461:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774311.739469:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774311.739473:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774311.739477:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774311.739481:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774311.739484:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774311.739489:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774311.739494:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774311.739498:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774311.739505:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774311.739509:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774311.739513:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774311.739518:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774311.739523:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774311.739527:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774311.739534:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775774311.739537:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774311.739542:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774311.739544:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774311.739546:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774311.739549:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774311.739551:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774311.739556:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775774311.739560:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774311.739568:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710254976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774311.739592:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774311.739594:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774311.739607:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710254976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774311.739616:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774311.739620:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774311.739623:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774311.739626:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774311.739627:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774311.739629:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774311.739633:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11194 00000400:00000200:2.0:1775774311.739638:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774311.739642:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774311.739645:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774311.739647:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774311.739648:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774311.739652:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774311.739655:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774311.739657:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774311.739657:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774311.739658:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774311.739660:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774311.739662:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774311.739666:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774311.739669:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774311.739671:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774311.739675:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774311.739677:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774311.739680:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774311.739685:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774311.739688:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774311.739692:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774311.739695:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774311.739697:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774311.739700:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774311.739702:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774311.739707:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774311.739709:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774311.739714:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775774311.739716:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774311.739718:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774311.739721:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774311.739724:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774311.739726:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774311.739729:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774311.739734:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775774311.739737:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774311.739742:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710255104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774311.739749:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774311.739751:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774311.739755:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710255104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774311.739764:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774311.739768:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774311.739772:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:3.1:1775774311.739772:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775774311.739774:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774311.739779:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:3.1:1775774311.739779:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000100:00000040:2.0:1775774311.739785:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11204) 00000800:00000200:3.1:1775774311.740057:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774311.740058:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1775774311.740237:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774311.740239:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000020:00000001:2.0:1775774312.891296:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774312.891299:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774312.891301:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775774316.859837:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774316.860106:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774316.860264:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:2.0:1775774318.011343:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774318.011346:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774318.011348:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775774319.931350:0:2399:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:2.0:1775774319.931385:0:2399:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1775774319.931394:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774319.931398:0:2399:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:2.0:1775774319.931402:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774319.931406:0:2399:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:2.0:1775774319.931412:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775774319.931416:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101298200. 00000400:00000200:2.0:1775774319.931421:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775774319.931425:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775774319.931431:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62079b27f8 00000400:00000010:2.0:1775774319.931434:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62079b27f8. 00000100:00000001:2.0:1775774319.931438:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775774319.931447:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6234918000 x1862022710255232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775774341 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774319.931478:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775774319.931483:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918000 x1862022710255232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775774341 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774319.931490:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774319.931492:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775774319.931502:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775774319.931505:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101298a00. 00000400:00000200:2.0:1775774319.931508:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775774319.931517:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775774319.931520:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62079b2e58 00000400:00000010:2.0:1775774319.931524:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62079b2e58. 00000100:00000001:2.0:1775774319.931529:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775774319.931534:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6234918a80 x1862022710255360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775774341 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774319.931554:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775774319.931560:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918a80 x1862022710255360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775774341 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774319.931571:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774319.931574:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775774319.931582:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775774319.931585:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101298c00. 00000400:00000200:2.0:1775774319.931589:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775774319.931597:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775774319.931602:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62079b2770 00000400:00000010:2.0:1775774319.931606:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62079b2770. 00000100:00000001:2.0:1775774319.931610:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775774319.931621:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f623491b800 x1862022710255488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775774341 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774319.931634:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775774319.931639:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491b800 x1862022710255488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775774341 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774319.931646:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774319.931650:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000200:3.1:1775774321.980026:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774321.980028:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775774321.980402:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774321.980405:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1775774321.980674:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774321.980676:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000020:00000001:2.0:1775774323.131436:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774323.131441:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774323.131444:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1775774327.099742:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774327.099746:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775774327.100093:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774327.100096:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1775774327.100383:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774327.100385:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000020:00000001:2.0:1775774328.251349:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774328.251352:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774328.251355:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774332.219766:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774332.219771:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774332.220171:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774332.220172:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774332.220372:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774332.220373:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000020:00000001:2.0:1775774333.371399:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774333.371402:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774333.371405:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774337.339281:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774337.339285:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774337.339288:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774337.339292:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774337.339295:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774337.339299:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774337.339301:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774337.339304:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774337.339306:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774337.339993:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774337.339998:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775774337.340445:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774337.340447:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775774337.340699:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774337.340700:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000100:00000001:2.0:1775774338.363235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774338.363243:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774338.363245:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774338.363251:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774338.363254:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774338.363258:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774338.363260:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774338.363262:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774338.363265:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774338.491292:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774338.491294:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774338.491296:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.0:1775774339.195269:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774339.195278:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774339.195282:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000100:00000001:2.0:1775774339.387263:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774339.387270:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774339.387272:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774339.387279:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774339.387283:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774339.387287:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774339.387290:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774339.387292:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774339.387294:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774340.411196:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774340.411202:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774340.411204:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774340.411209:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774340.411211:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774340.411215:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774340.411217:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774340.411218:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774340.411220:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774341.435247:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774341.435256:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774341.435258:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774341.435273:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774341.435277:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774341.435280:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774341.435283:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774341.435285:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774341.435287:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774342.459179:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774342.459185:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774342.459187:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774342.459191:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774342.459194:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774342.459197:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774342.459199:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774342.459201:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774342.459203:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774342.459771:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774342.459775:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775774342.460210:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774342.460212:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775774342.460475:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774342.460477:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000100:00000001:2.0:1775774343.483255:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774343.483262:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774343.483264:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774343.483269:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774343.483273:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774343.483276:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774343.483279:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774343.483281:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774343.483284:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774343.611324:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774343.611329:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774343.611332:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774344.507261:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774344.507267:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774344.507269:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774344.507274:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774344.507278:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774344.507283:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774344.507285:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774344.507287:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774344.507289:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774345.531260:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774345.531268:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774345.531270:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774345.531277:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774345.531282:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774345.531286:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774345.531289:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774345.531292:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774345.531295:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774346.555264:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774346.555271:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774346.555274:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774346.555281:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774346.555285:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774346.555291:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774346.555294:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774346.555296:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774346.555299:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774347.579244:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774347.579250:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774347.579254:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774347.579268:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774347.579277:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774347.579286:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774347.579296:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774347.579300:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774347.579304:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774347.579814:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774347.579817:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775774347.580086:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774347.580087:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775774347.580311:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774347.580312:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000100:00000001:2.0:1775774348.603188:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774348.603193:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774348.603195:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774348.603198:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774348.603201:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774348.603203:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774348.603206:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774348.603207:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774348.603209:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774348.731277:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774348.731280:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774348.731283:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774349.627237:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774349.627244:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774349.627246:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774349.627251:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774349.627255:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774349.627258:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774349.627261:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774349.627263:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774349.627265:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774350.651224:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774350.651229:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774350.651231:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774350.651235:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774350.651238:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774350.651241:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774350.651242:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774350.651244:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774350.651247:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774351.675237:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774351.675242:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774351.675244:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774351.675248:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774351.675250:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774351.675253:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774351.675255:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774351.675257:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774351.675259:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774352.699220:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774352.699226:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774352.699228:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774352.699233:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774352.699237:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774352.699241:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774352.699243:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774352.699245:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774352.699248:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774352.699933:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774352.699937:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775774352.700397:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774352.700400:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775774352.700640:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774352.700642:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000100:00000001:2.0:1775774353.723250:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774353.723256:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774353.723257:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774353.723262:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774353.723266:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774353.723270:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774353.723272:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774353.723275:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774353.723277:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774353.851346:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774353.851351:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774353.851355:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774354.747199:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774354.747204:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774354.747206:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774354.747210:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774354.747212:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774354.747215:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774354.747217:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774354.747219:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774354.747220:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774355.771185:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774355.771189:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774355.771191:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774355.771195:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774355.771197:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774355.771200:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774355.771202:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774355.771203:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774355.771205:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774356.795223:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774356.795232:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774356.795236:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774356.795242:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774356.795250:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774356.795253:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774356.795257:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774356.795259:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774356.795262:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774357.819232:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774357.819238:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774357.819242:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774357.819248:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774357.819252:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774357.819256:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774357.819258:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774357.819260:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774357.819264:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774357.819999:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774357.820003:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774357.820403:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774357.820405:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774357.820589:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774357.820591:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1775774358.843195:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774358.843202:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774358.843204:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774358.843209:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774358.843212:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774358.843215:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774358.843217:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774358.843218:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774358.843220:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774358.971316:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774358.971318:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774358.971321:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774359.867224:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774359.867229:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774359.867230:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774359.867235:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774359.867237:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774359.867240:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774359.867242:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774359.867244:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774359.867245:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774360.891270:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774360.891279:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774360.891280:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774360.891289:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774360.891294:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774360.891298:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774360.891302:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774360.891305:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774360.891308:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774361.915217:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774361.915224:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774361.915226:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774361.915231:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774361.915234:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774361.915238:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774361.915241:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774361.915243:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774361.915246:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774362.939267:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774362.939273:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774362.939276:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774362.939281:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774362.939285:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774362.939289:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774362.939292:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774362.939296:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774362.939298:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774362.939813:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774362.939819:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774362.940277:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774362.940278:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774362.940478:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774362.940480:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1775774363.963190:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774363.963195:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774363.963196:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774363.963200:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774363.963203:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774363.963205:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774363.963207:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774363.963209:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774363.963210:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774364.091313:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774364.091316:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774364.091319:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774364.987250:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774364.987256:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774364.987259:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774364.987265:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774364.987268:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774364.987272:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774364.987275:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774364.987277:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774364.987280:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774366.011236:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774366.011241:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774366.011243:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774366.011247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774366.011250:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774366.011254:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774366.011256:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774366.011258:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774366.011261:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774367.035249:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774367.035256:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774367.035258:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774367.035264:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774367.035267:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774367.035270:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774367.035273:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774367.035275:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774367.035278:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774368.059271:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774368.059279:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774368.059284:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774368.059290:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774368.059296:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774368.059301:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774368.059304:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774368.059309:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774368.059314:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774368.059828:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774368.059832:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774368.060214:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774368.060216:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774368.060480:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774368.060482:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000100:00000001:2.0:1775774369.083279:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774369.083286:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774369.083287:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774369.083293:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774369.083297:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774369.083301:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774369.083303:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774369.083305:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774369.083307:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774369.211359:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774369.211362:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774369.211365:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774370.107253:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774370.107261:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774370.107262:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774370.107268:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774370.107271:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774370.107275:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774370.107277:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774370.107280:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774370.107282:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774371.131284:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774371.131294:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774371.131297:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774371.131305:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774371.131309:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774371.131314:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774371.131317:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774371.131320:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774371.131324:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774372.155229:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774372.155236:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774372.155238:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774372.155243:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774372.155247:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774372.155251:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774372.155253:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774372.155255:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774372.155258:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774373.179287:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774373.179296:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774373.179299:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774373.179307:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774373.179313:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774373.179318:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774373.179325:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774373.179329:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774373.179335:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774373.179808:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774373.179813:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774373.180228:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774373.180229:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774373.180451:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774373.180453:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000100:00000001:2.0:1775774374.203241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774374.203251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774374.203254:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774374.203261:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774374.203265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774374.203270:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774374.203273:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774374.203276:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774374.203279:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774374.331385:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774374.331390:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774374.331393:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774375.227284:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774375.227294:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774375.227297:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774375.227304:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774375.227308:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774375.227313:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774375.227316:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774375.227319:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774375.227322:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774376.251194:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774376.251201:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774376.251203:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774376.251208:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774376.251211:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774376.251215:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774376.251217:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774376.251219:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774376.251221:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775774377.083280:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774377.083289:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774377.083297:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000100:00000001:2.0:1775774377.275247:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774377.275254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774377.275256:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774377.275261:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774377.275265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774377.275269:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774377.275271:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774377.275273:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774377.275275:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774378.299233:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774378.299242:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774378.299244:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774378.299250:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774378.299255:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774378.299259:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774378.299262:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774378.299264:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774378.299267:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774378.299737:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774378.299740:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775774378.300014:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774378.300016:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775774378.300198:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774378.300200:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000100:00000001:2.0:1775774379.323223:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774379.323229:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774379.323231:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774379.323237:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774379.323240:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774379.323244:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774379.323246:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774379.323248:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774379.323250:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774379.451332:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774379.451335:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774379.451338:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774380.347234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774380.347240:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774380.347243:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774380.347247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774380.347252:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774380.347255:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774380.347259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774380.347261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774380.347264:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774381.371212:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774381.371222:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774381.371224:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774381.371231:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774381.371236:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774381.371239:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774381.371242:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774381.371244:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774381.371246:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774382.395266:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774382.395275:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774382.395277:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774382.395285:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774382.395289:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774382.395294:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774382.395298:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774382.395301:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774382.395304:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774383.419249:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774383.419256:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774383.419257:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774383.419263:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774383.419266:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774383.419270:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774383.419274:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774383.419280:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774383.419284:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1775774383.419755:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774383.420008:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:3.1:1775774383.420183:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:2.0:1775774384.443304:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774384.443310:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774384.443311:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774384.443316:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774384.443318:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774384.443322:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774384.443324:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774384.443326:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774384.443351:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774384.571278:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774384.571281:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774384.571284:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774385.467280:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774385.467285:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774385.467287:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774385.467292:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774385.467295:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774385.467299:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774385.467301:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774385.467303:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774385.467306:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774386.491382:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774386.491388:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774386.491389:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774386.491394:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774386.491397:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774386.491401:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774386.491404:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774386.491406:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774386.491408:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774387.515362:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774387.515370:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774387.515372:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774387.515382:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774387.515386:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774387.515389:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774387.515392:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774387.515395:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774387.515398:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774388.539323:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774388.539329:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774388.539332:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774388.539338:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774388.539342:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774388.539345:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774388.539348:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774388.539352:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774388.539361:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774388.540110:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774388.540122:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774388.540669:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774388.540673:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774388.540921:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774388.540922:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000100:00000001:2.0:1775774389.563393:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774389.563400:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774389.563402:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774389.563407:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774389.563410:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774389.563414:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774389.563417:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774389.563418:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774389.563421:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774389.691350:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774389.691357:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774389.691362:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774390.587214:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774390.587238:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774390.587240:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774390.587249:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774390.587253:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774390.587256:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774390.587259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774390.587261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774390.587264:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774391.611232:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774391.611242:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774391.611245:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774391.611253:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774391.611259:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774391.611264:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774391.611267:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774391.611271:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774391.611274:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774392.635263:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774392.635270:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774392.635272:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774392.635278:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774392.635282:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774392.635285:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774392.635288:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774392.635290:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774392.635292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774393.659206:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774393.659213:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774393.659215:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774393.659220:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774393.659224:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774393.659227:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774393.659230:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774393.659232:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774393.659235:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774393.659732:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774393.659737:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774393.660092:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774393.660093:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774393.660296:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774393.660297:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000100:00000001:2.0:1775774394.683212:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774394.683217:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774394.683219:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774394.683223:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774394.683225:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774394.683228:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774394.683230:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774394.683232:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774394.683234:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774394.811254:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774394.811257:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774394.811259:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774395.707279:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774395.707287:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774395.707289:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774395.707294:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774395.707297:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774395.707301:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774395.707302:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774395.707304:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774395.707306:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774396.731198:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774396.731203:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774396.731204:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774396.731209:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774396.731212:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774396.731215:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774396.731217:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774396.731219:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774396.731221:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774397.755254:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774397.755261:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774397.755264:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774397.755270:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774397.755274:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774397.755278:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774397.755281:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774397.755283:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774397.755286:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774398.779200:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774398.779205:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774398.779208:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774398.779214:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774398.779218:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774398.779222:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774398.779225:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774398.779228:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774398.779233:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774398.779694:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774398.779697:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774398.779987:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774398.779988:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774398.780237:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774398.780240:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1775774399.803230:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774399.803238:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774399.803240:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774399.803247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774399.803251:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774399.803255:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774399.803257:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774399.803260:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774399.803262:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774399.931347:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774399.931351:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774399.931355:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774400.827252:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774400.827259:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774400.827261:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774400.827267:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774400.827272:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774400.827281:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774400.827283:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774400.827286:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774400.827288:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774401.851269:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774401.851276:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774401.851278:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774401.851283:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774401.851287:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774401.851291:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774401.851294:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774401.851297:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774401.851299:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774402.875256:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774402.875266:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774402.875268:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774402.875275:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774402.875278:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774402.875282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774402.875285:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774402.875288:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774402.875291:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774403.899272:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774403.899279:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774403.899283:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774403.899291:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774403.899300:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774403.899309:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774403.899319:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774403.899323:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774403.899332:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774403.899808:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774403.899812:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774403.900165:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774403.900167:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774403.900377:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774403.900379:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000100:00000001:2.0:1775774404.923227:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774404.923237:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774404.923240:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774404.923248:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774404.923253:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774404.923258:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774404.923262:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774404.923266:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774404.923269:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774405.051356:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774405.051361:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774405.051365:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774405.947262:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774405.947269:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774405.947271:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774405.947277:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774405.947281:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774405.947284:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774405.947287:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774405.947289:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774405.947292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774406.971225:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774406.971231:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774406.971233:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774406.971237:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774406.971240:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774406.971244:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774406.971246:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774406.971248:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774406.971250:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774407.995243:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774407.995250:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774407.995252:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774407.995259:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774407.995262:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774407.995266:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774407.995270:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774407.995272:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774407.995275:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774409.019225:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774409.019231:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774409.019233:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774409.019238:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774409.019241:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774409.019245:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774409.019248:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774409.019251:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774409.019254:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774409.019829:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774409.019837:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775774409.020372:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774409.020374:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775774409.020674:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774409.020676:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000100:00000001:2.0:1775774410.043262:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774410.043268:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774410.043269:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774410.043274:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774410.043278:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774410.043282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774410.043285:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774410.043287:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774410.043290:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774410.171319:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774410.171323:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774410.171327:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774411.067197:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774411.067202:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774411.067203:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774411.067208:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774411.067211:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774411.067213:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774411.067215:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774411.067217:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774411.067219:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774412.091265:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774412.091271:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774412.091272:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774412.091277:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774412.091280:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774412.091282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774412.091284:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774412.091286:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774412.091288:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774413.115237:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774413.115243:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774413.115244:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774413.115249:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774413.115253:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774413.115256:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774413.115258:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774413.115260:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774413.115262:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774414.139218:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774414.139224:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774414.139225:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774414.139230:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774414.139233:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774414.139236:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774414.139238:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774414.139240:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774414.139243:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774414.139778:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774414.139785:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775774414.140259:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774414.140261:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775774414.140481:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774414.140483:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000100:00000001:2.0:1775774415.163228:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774415.163237:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774415.163239:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774415.163246:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774415.163251:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774415.163256:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774415.163259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774415.163262:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774415.163266:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774415.291295:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774415.291299:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774415.291302:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.0:1775774415.995282:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774415.995293:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774415.995300:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000100:00000001:2.0:1775774416.187256:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774416.187266:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774416.187268:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774416.187276:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774416.187281:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774416.187287:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774416.187290:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774416.187294:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774416.187298:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774417.211294:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774417.211306:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774417.211308:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774417.211316:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774417.211321:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774417.211326:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774417.211329:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774417.211332:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774417.211335:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774419.259927:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774419.259931:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775774419.260328:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774419.260329:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775774419.260518:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774419.260520:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000020:00000001:2.0:1775774420.411448:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774420.411452:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774420.411456:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774424.379897:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774424.379902:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774424.380329:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774424.380332:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774424.380608:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774424.380610:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000020:00000001:2.0:1775774425.531322:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774425.531324:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774425.531327:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774429.499667:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774429.499674:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774429.499947:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774429.499948:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774429.500176:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774429.500177:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1775774430.651324:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774430.651327:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774430.651329:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774434.619831:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774434.619833:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774434.620189:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774434.620190:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774434.620464:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774434.620467:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000020:00000001:2.0:1775774435.771279:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774435.771282:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774435.771285:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774436.795285:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774436.795288:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774436.795291:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774439.739713:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774439.739718:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774439.740062:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774439.740064:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774439.740292:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774439.740293:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000020:00000001:2.0:1775774440.891328:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774440.891331:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774440.891333:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774441.915312:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774441.915315:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774441.915317:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774444.859244:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774444.859254:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774444.859257:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774444.859267:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774444.859298:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774444.859306:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11327 00000400:00000200:2.0:1775774444.859315:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774444.859320:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774444.859323:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774444.859327:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774444.859330:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774444.859334:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774444.859336:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774444.859339:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774444.859340:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774444.859342:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774444.859344:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774444.859345:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774444.859347:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774444.859349:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774444.859350:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774444.859351:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774444.859352:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774444.859354:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774444.859356:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774444.859357:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774444.859359:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774444.859361:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774444.859362:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774444.859364:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774444.859365:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774444.859373:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775774444.859375:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774444.859377:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774444.859378:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774444.859380:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774444.859382:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774444.859386:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774444.859391:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775774444.859396:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774444.859402:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710264832/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774444.859409:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774444.859413:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774444.859415:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710264832/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774444.859433:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774444.859435:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774444.859437:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774444.859438:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774444.859439:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774444.859441:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774444.859443:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11327 00000400:00000200:2.0:1775774444.859445:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774444.859446:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774444.859448:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774444.859450:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774444.859451:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774444.859454:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774444.859456:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774444.859458:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774444.859459:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774444.859461:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774444.859463:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774444.859465:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774444.859467:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774444.859468:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774444.859470:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774444.859473:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774444.859478:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774444.859484:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774444.859495:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774444.859497:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774444.859502:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774444.859507:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774444.859518:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774444.859520:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774444.859523:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774444.859533:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774444.859543:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774444.859546:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775774444.859553:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774444.859557:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774444.859558:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774444.859561:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774444.859567:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774444.859574:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774444.859581:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775774444.859586:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774444.859595:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710264960/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774444.859615:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774444.859616:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774444.859620:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710264960/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774444.859656:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774444.859665:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774444.859667:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774444.859669:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774444.859671:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774444.859676:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774444.859678:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11327 00000400:00000200:2.0:1775774444.859686:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774444.859694:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774444.859698:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774444.859707:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774444.859710:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774444.859715:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774444.859718:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774444.859722:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774444.859725:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774444.859729:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774444.859733:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774444.859740:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774444.859745:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774444.859758:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774444.859765:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774444.859771:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774444.859773:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774444.859774:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774444.859779:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774444.859782:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774444.859792:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774444.859797:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774444.859801:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774444.859805:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774444.859807:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774444.859811:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774444.859813:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774444.859816:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775774444.859818:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774444.859825:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774444.859827:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774444.859828:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774444.859829:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774444.859831:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774444.859840:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775774444.859848:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774444.859857:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710265088/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774444.859878:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774444.859882:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774444.859888:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710265088/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774444.859910:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774444.859913:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774444.859915:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774444.859917:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774444.859919:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774444.859924:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11337) 00000800:00000200:1.1:1775774444.859961:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774444.859964:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774444.860336:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774444.860338:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774444.860533:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774444.860535:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000020:00000001:2.0:1775774446.011300:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774446.011303:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774446.011305:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774447.035291:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774447.035293:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774447.035295:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774449.979223:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774449.979232:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774449.979236:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774449.979241:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774449.979246:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774449.979255:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11332 00000400:00000200:2.0:1775774449.979266:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774449.979273:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774449.979277:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774449.979280:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774449.979284:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774449.979286:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774449.979288:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774449.979291:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774449.979293:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774449.979295:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774449.979298:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774449.979300:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774449.979302:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774449.979305:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774449.979307:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774449.979309:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774449.979310:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774449.979313:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774449.979316:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774449.979318:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774449.979321:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774449.979324:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774449.979327:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774449.979329:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774449.979331:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774449.979339:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775774449.979343:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774449.979347:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774449.979349:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774449.979352:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774449.979354:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774449.979358:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774449.979365:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775774449.979371:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774449.979380:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710265216/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774449.979390:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774449.979394:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774449.979399:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710265216/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774449.979420:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774449.979423:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774449.979425:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774449.979427:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774449.979429:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774449.979431:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774449.979433:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11332 00000400:00000200:2.0:1775774449.979437:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774449.979441:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774449.979444:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774449.979448:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774449.979450:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774449.979452:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774449.979455:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774449.979458:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774449.979461:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774449.979463:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774449.979466:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774449.979471:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774449.979476:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774449.979482:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774449.979484:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774449.979489:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774449.979494:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774449.979498:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774449.979505:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774449.979509:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774449.979512:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774449.979518:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774449.979524:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774449.979527:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774449.979532:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774449.979537:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774449.979540:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774449.979545:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775774449.979550:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774449.979556:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774449.979558:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774449.979563:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774449.979566:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774449.979570:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774449.979577:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775774449.979582:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774449.979592:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710265344/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774449.979610:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774449.979614:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774449.979622:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710265344/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774449.979654:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774449.979659:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774449.979666:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774449.979669:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774449.979671:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774449.979675:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774449.979680:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11332 00000400:00000200:2.0:1775774449.979687:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774449.979694:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774449.979699:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000800:00000200:1.1:1775774449.979699:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775774449.979705:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774449.979708:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774449.979711:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774449.979712:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774449.979715:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774449.979716:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774449.979718:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774449.979721:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774449.979726:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774449.979730:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774449.979736:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774449.979740:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774449.979743:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774449.979748:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774449.979750:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774449.979756:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774449.979759:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774449.979762:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774449.979766:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774449.979769:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774449.979772:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774449.979777:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774449.979781:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774449.979785:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774449.979791:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775774449.979794:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774449.979799:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774449.979801:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774449.979805:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774449.979808:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774449.979812:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774449.979819:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775774449.979825:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774449.979835:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710265472/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774449.979850:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774449.979854:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774449.979862:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710265472/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774449.979875:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774449.979880:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774449.979883:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774449.979887:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774449.979894:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774449.979903:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11342) 00000800:00000200:1.1:1775774449.979970:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774449.980109:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:2.0:1775774451.131279:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774451.131282:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774451.131285:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774452.155283:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774452.155286:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774452.155290:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774455.099222:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774455.099230:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774455.099233:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774455.099237:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774455.099241:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774455.099248:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11337 00000400:00000200:2.0:1775774455.099256:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774455.099264:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774455.099267:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774455.099270:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774455.099273:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774455.099275:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774455.099277:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774455.099279:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774455.099281:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774455.099283:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774455.099285:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774455.099287:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774455.099288:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774455.099291:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774455.099292:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774455.099293:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774455.099294:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774455.099296:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774455.099298:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774455.099299:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774455.099302:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774455.099305:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774455.099307:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774455.099309:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774455.099311:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774455.099332:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775774455.099336:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774455.099340:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774455.099342:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774455.099344:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774455.099347:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774455.099351:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774455.099361:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775774455.099371:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774455.099379:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710265600/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774455.099390:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774455.099394:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774455.099398:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710265600/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774455.099418:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774455.099420:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774455.099423:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774455.099424:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774455.099425:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774455.099427:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774455.099429:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11337 00000400:00000200:2.0:1775774455.099432:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774455.099434:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774455.099436:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774455.099438:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774455.099439:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774455.099441:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774455.099442:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774455.099444:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774455.099445:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774455.099447:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774455.099448:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774455.099450:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774455.099451:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774455.099453:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774455.099454:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774455.099455:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774455.099456:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774455.099458:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774455.099460:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774455.099462:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774455.099466:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774455.099471:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774455.099474:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774455.099476:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774455.099480:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774455.099484:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774455.099486:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774455.099492:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775774455.099503:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774455.099507:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774455.099510:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774455.099515:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774455.099518:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774455.099524:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774455.099529:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775774455.099532:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774455.099544:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710265728/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774455.099610:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774455.099619:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774455.099631:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710265728/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774455.099667:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774455.099676:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774455.099681:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774455.099683:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774455.099684:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774455.099691:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774455.099705:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11337 00000400:00000200:2.0:1775774455.099712:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774455.099717:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774455.099728:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774455.099736:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774455.099741:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774455.099757:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774455.099761:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774455.099767:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774455.099774:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774455.099778:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774455.099780:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774455.099782:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774455.099787:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774455.099795:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774455.099798:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774455.099801:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774455.099812:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774455.099825:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774455.099839:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774455.099856:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774455.099869:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774455.099879:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774455.099887:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774455.099895:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774455.099901:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774455.099907:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774455.099916:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774455.099927:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775774455.099931:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774455.099935:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774455.099941:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774455.099947:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774455.099949:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774455.099951:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774455.099956:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775774455.099967:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774455.099979:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710265856/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774455.100002:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774455.100014:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774455.100027:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710265856/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774455.100058:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774455.100064:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774455.100073:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774455.100085:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774455.100110:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774455.100120:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11347) 00000800:00000200:0.1:1775774455.100283:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774455.100287:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775774455.100769:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774455.100773:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775774455.101108:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774455.101111:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000020:00000001:2.0:1775774456.251321:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774456.251323:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774456.251326:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774457.275319:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774457.275322:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774457.275325:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774460.219270:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774460.219278:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774460.219281:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774460.219287:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774460.219292:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774460.219296:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774460.219301:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774460.219304:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774460.219308:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775774460.219872:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774460.219879:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775774460.220382:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774460.220384:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775774460.220611:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774460.220613:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000100:00000001:2.0:1775774461.243197:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774461.243204:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774461.243206:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774461.243212:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774461.243215:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774461.243220:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774461.243222:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774461.243225:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774461.243227:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774461.371337:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774461.371340:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774461.371344:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774462.267274:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774462.267283:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774462.267285:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774462.267292:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774462.267296:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774462.267299:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774462.267303:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774462.267305:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774462.267307:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774462.395291:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774462.395297:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774462.395302:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774463.291260:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774463.291268:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774463.291270:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774463.291275:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774463.291279:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774463.291283:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774463.291286:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774463.291288:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774463.291290:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774464.315262:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774464.315270:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774464.315272:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774464.315277:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774464.315281:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774464.315286:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774464.315288:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774464.315291:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774464.315293:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774465.339277:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774465.339286:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774465.339292:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774465.339302:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774465.339309:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774465.339319:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774465.339325:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774465.339329:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774465.339333:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775774465.339798:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774465.339805:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775774465.340371:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774465.340374:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775774465.340670:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774465.340673:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000100:00000001:2.0:1775774466.363240:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774466.363248:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774466.363249:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774466.363256:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774466.363270:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774466.363275:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774466.363278:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774466.363280:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774466.363283:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774466.491299:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774466.491303:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774466.491306:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774467.387210:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774467.387216:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774467.387218:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774467.387222:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774467.387225:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774467.387228:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774467.387230:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774467.387232:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774467.387235:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774467.515362:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774467.515367:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774467.515371:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774468.411204:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774468.411210:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774468.411214:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774468.411220:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774468.411223:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774468.411227:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774468.411230:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774468.411233:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774468.411235:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774469.435237:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774469.435242:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774469.435244:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774469.435248:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774469.435251:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774469.435254:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774469.435256:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774469.435257:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774469.435259:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774470.459280:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774470.459287:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774470.459290:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774470.459297:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774470.459305:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774470.459313:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774470.459318:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774470.459324:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774470.459330:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775774470.460033:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774470.460041:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775774470.460490:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774470.460492:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775774470.460773:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774470.460776:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000100:00000001:2.0:1775774471.483238:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774471.483245:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774471.483247:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774471.483253:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774471.483256:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774471.483260:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774471.483263:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774471.483266:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774471.483269:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774471.611340:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774471.611344:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774471.611348:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774472.507272:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774472.507283:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774472.507285:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774472.507293:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774472.507298:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774472.507303:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774472.507306:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774472.507310:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774472.507313:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774472.635356:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774472.635361:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774472.635365:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774475.579790:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774475.579796:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775774475.580295:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774475.580298:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775774475.580614:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774475.580617:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000020:00000001:2.0:1775774476.731444:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774476.731449:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774476.731454:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774477.755311:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774477.755316:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774477.755320:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774480.699821:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774480.699825:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775774480.700385:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774480.700388:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775774480.700663:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774480.700665:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000020:00000001:2.0:1775774481.851269:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774481.851273:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774481.851275:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774482.875370:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774482.875374:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774482.875377:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774485.819771:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774485.819777:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775774485.820205:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774485.820207:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775774485.820388:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774485.820390:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000020:00000001:2.0:1775774486.971314:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774486.971318:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774486.971320:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774487.995313:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774487.995317:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774487.995320:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774490.939253:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774490.939263:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774490.939268:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774490.939273:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774490.939281:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774490.939288:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11373 00000400:00000200:2.0:1775774490.939298:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774490.939305:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774490.939310:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774490.939315:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774490.939320:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774490.939325:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774490.939330:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774490.939333:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774490.939335:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774490.939338:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774490.939340:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774490.939341:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774490.939343:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774490.939345:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774490.939347:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774490.939349:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774490.939352:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774490.939355:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774490.939358:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774490.939359:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774490.939362:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774490.939364:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774490.939366:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774490.939368:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774490.939369:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774490.939377:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775774490.939380:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774490.939383:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774490.939384:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774490.939388:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774490.939390:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774490.939401:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774490.939411:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775774490.939418:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774490.939428:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710268288/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774490.939439:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774490.939443:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774490.939449:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710268288/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774490.939485:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774490.939489:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774490.939493:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774490.939494:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774490.939496:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774490.939499:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774490.939503:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11373 00000400:00000200:2.0:1775774490.939508:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774490.939513:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774490.939516:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774490.939519:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774490.939523:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774490.939530:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774490.939535:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774490.939541:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774490.939543:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774490.939548:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774490.939553:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774490.939564:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774490.939573:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774490.939581:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774490.939589:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774490.939597:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774490.939601:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774490.939608:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774490.939617:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774490.939625:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774490.939630:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774490.939639:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774490.939648:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774490.939655:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774490.939662:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774490.939666:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774490.939669:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774490.939678:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775774490.939683:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774490.939690:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774490.939695:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774490.939698:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774490.939702:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774490.939707:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774490.939714:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775774490.939722:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774490.939730:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710268416/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774490.939746:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774490.939753:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774490.939765:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710268416/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774490.939807:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774490.939814:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774490.939820:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774490.939822:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774490.939824:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774490.939833:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774490.939843:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11373 00000400:00000200:2.0:1775774490.939856:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774490.939869:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774490.939880:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774490.939889:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774490.939898:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774490.939904:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774490.939910:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774490.939916:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774490.939920:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774490.939924:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774490.939927:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774490.939931:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774490.939937:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774490.939951:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774490.939954:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774490.939964:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774490.939967:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774490.939970:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774490.939978:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774490.939984:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774490.939991:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:0.1:1775774490.940002:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:2.0:1775774490.940003:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:0.1:1775774490.940008:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 02000000:00000001:2.0:1775774490.940010:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774490.940019:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774490.940025:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774490.940031:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774490.940033:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774490.940042:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775774490.940048:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774490.940051:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774490.940054:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774490.940057:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774490.940058:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774490.940061:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774490.940066:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775774490.940070:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774490.940078:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710268544/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774490.940171:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774490.940173:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774490.940178:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710268544/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774490.940198:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774490.940201:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774490.940206:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774490.940214:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774490.940222:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774490.940229:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11383) 00000800:00000200:0.1:1775774490.940592:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774490.940595:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775774490.940852:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774490.940854:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000020:00000001:2.0:1775774492.091381:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774492.091385:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774492.091389:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774493.115353:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774493.115357:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774493.115360:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774496.059231:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774496.059244:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774496.059248:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774496.059254:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774496.059259:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774496.059268:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11378 00000400:00000200:2.0:1775774496.059284:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774496.059298:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774496.059306:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774496.059313:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774496.059320:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774496.059325:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774496.059334:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774496.059343:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774496.059347:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774496.059351:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774496.059354:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774496.059358:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774496.059361:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774496.059366:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774496.059369:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774496.059372:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774496.059374:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774496.059377:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774496.059400:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774496.059403:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774496.059407:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774496.059413:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774496.059417:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774496.059421:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774496.059424:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774496.059437:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775774496.059441:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774496.059447:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774496.059449:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774496.059453:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774496.059456:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774496.059461:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774496.059472:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775774496.059479:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774496.059490:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710268672/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774496.059502:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774496.059507:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774496.059513:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710268672/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774496.059541:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774496.059544:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774496.059548:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774496.059550:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774496.059552:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774496.059556:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774496.059558:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11378 00000400:00000200:2.0:1775774496.059563:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774496.059567:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774496.059569:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774496.059572:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774496.059575:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774496.059578:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774496.059580:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774496.059583:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774496.059584:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774496.059586:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774496.059589:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774496.059594:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774496.059598:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774496.059605:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774496.059610:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774496.059615:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774496.059620:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774496.059627:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774496.059632:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774496.059636:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774496.059639:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774496.059647:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774496.059651:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774496.059655:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774496.059662:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774496.059667:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774496.059671:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774496.059676:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775774496.059681:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774496.059685:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774496.059688:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774496.059689:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774496.059691:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774496.059693:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774496.059699:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775774496.059703:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774496.059713:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710268800/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774496.059727:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774496.059730:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774496.059736:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710268800/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774496.059751:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774496.059755:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774496.059765:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774496.059769:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774496.059773:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774496.059779:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774496.059786:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11378 00000400:00000200:2.0:1775774496.059795:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774496.059801:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774496.059806:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774496.059810:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774496.059812:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774496.059814:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774496.059820:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774496.059825:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774496.059827:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774496.059829:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774496.059833:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774496.059838:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774496.059841:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774496.059847:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774496.059850:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774496.059851:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774496.059853:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774496.059855:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774496.059860:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774496.059863:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774496.059866:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774496.059870:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774496.059874:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774496.059879:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774496.059882:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774496.059887:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774496.059892:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774496.059898:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775774496.059902:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774496.059905:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774496.059908:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774496.059913:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774496.059917:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774496.059919:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774496.059925:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775774496.059931:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774496.059938:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710268928/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774496.059953:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774496.059958:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774496.059964:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710268928/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1775774496.059971:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774496.059977:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1775774496.059981:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774496.059987:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774496.059995:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774496.059999:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774496.060007:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774496.060015:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11388) 00000800:00000200:0.1:1775774496.060450:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774496.060453:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775774496.060704:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774496.060706:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1775774497.211342:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774497.211349:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774497.211354:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774498.235370:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774498.235375:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774498.235379:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774501.179270:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774501.179310:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774501.179315:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774501.179321:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774501.179329:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774501.179339:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11383 00000400:00000200:2.0:1775774501.179349:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774501.179357:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774501.179361:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774501.179364:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774501.179367:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774501.179369:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774501.179371:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774501.179374:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774501.179376:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774501.179379:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774501.179382:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774501.179384:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774501.179386:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774501.179388:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774501.179389:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774501.179390:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774501.179391:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774501.179394:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774501.179396:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774501.179398:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774501.179401:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774501.179403:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774501.179405:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774501.179407:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774501.179409:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774501.179419:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775774501.179422:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774501.179426:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774501.179427:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774501.179429:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774501.179432:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774501.179436:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774501.179445:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775774501.179451:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774501.179460:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710269056/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774501.179469:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774501.179474:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774501.179477:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710269056/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774501.179497:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774501.179500:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774501.179503:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774501.179504:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774501.179505:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774501.179508:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774501.179509:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11383 00000400:00000200:2.0:1775774501.179512:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774501.179515:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774501.179517:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774501.179519:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774501.179522:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774501.179523:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774501.179525:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774501.179527:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774501.179528:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774501.179529:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774501.179531:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774501.179533:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774501.179536:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774501.179539:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774501.179541:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774501.179544:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774501.179546:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774501.179548:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774501.179551:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774501.179556:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774501.179558:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774501.179565:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774501.179571:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774501.179573:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774501.179580:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774501.179585:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774501.179590:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774501.179596:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775774501.179600:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774501.179605:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774501.179608:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774501.179609:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774501.179610:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774501.179612:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774501.179616:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775774501.179622:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774501.179628:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710269184/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774501.179637:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774501.179639:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774501.179642:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710269184/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774501.179651:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774501.179653:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774501.179655:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774501.179657:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774501.179660:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774501.179663:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774501.179667:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11383 00000400:00000200:2.0:1775774501.179673:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774501.179676:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774501.179677:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774501.179682:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774501.179685:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774501.179688:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774501.179692:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774501.179695:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774501.179697:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774501.179700:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774501.179702:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774501.179704:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774501.179708:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774501.179713:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774501.179716:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774501.179718:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774501.179723:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774501.179724:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774501.179729:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774501.179731:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774501.179733:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774501.179735:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774501.179738:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774501.179740:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774501.179741:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774501.179743:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774501.179744:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774501.179747:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775774501.179750:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774501.179752:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774501.179753:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774501.179754:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774501.179756:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774501.179766:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774501.179773:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775774501.179780:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774501.179789:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710269312/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774501.179824:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774501.179827:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774501.179834:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710269312/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774501.179846:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774501.179849:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774501.179854:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:0.1:1775774501.179857:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775774501.179858:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774501.179864:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:0.1:1775774501.179864:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000100:00000040:2.0:1775774501.179869:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11393) 00000800:00000200:0.1:1775774501.180316:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774501.180317:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775774501.180514:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774501.180515:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000020:00000001:2.0:1775774502.331328:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774502.331336:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774502.331341:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774503.355464:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774503.355470:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774503.355476:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774506.299837:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774506.299843:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775774506.300395:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774506.300398:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775774506.300690:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774506.300693:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000020:00000001:2.0:1775774507.451371:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774507.451378:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774507.451383:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774508.475367:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774508.475372:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774508.475376:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774511.419810:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774511.419816:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775774511.420295:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774511.420298:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775774511.420591:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774511.420594:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000020:00000001:2.0:1775774512.571334:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774512.571339:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774512.571343:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774513.595294:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774513.595299:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774513.595303:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774516.539328:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774516.539348:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774516.539351:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774516.539356:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774516.539361:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774516.539370:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11399 00000400:00000200:2.0:1775774516.539382:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774516.539388:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774516.539392:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774516.539395:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774516.539399:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774516.539402:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774516.539404:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774516.539408:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774516.539410:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774516.539413:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774516.539416:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774516.539419:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774516.539421:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774516.539425:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774516.539426:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774516.539428:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774516.539430:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774516.539433:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774516.539437:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774516.539439:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774516.539443:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774516.539446:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774516.539450:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774516.539452:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774516.539454:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774516.539465:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775774516.539469:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774516.539473:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774516.539475:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774516.539478:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774516.539480:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774516.539484:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774516.539492:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775774516.539499:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774516.539508:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710270208/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774516.539520:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774516.539524:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774516.539529:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710270208/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774516.539560:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774516.539563:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774516.539567:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774516.539569:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774516.539570:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774516.539574:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774516.539576:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11399 00000400:00000200:2.0:1775774516.539581:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774516.539586:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774516.539589:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774516.539594:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774516.539596:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774516.539598:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774516.539601:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774516.539606:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774516.539610:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774516.539616:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774516.539620:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774516.539625:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774516.539632:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774516.539649:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774516.539660:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774516.539666:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774516.539673:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774516.539684:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774516.539692:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774516.539699:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774516.539706:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774516.539713:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774516.539723:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774516.539729:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774516.539735:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774516.539740:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774516.539745:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774516.539753:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775774516.539759:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774516.539764:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774516.539766:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774516.539771:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774516.539775:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774516.539779:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774516.539787:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775774516.539794:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774516.539808:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710270336/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774516.539836:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774516.539843:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774516.539852:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710270336/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774516.539916:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774516.539923:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774516.539938:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774516.539943:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774516.539948:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774516.539952:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774516.539958:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11399 00000400:00000200:2.0:1775774516.539973:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774516.539984:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774516.539992:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774516.539998:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774516.540003:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774516.540008:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774516.540011:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774516.540020:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774516.540035:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774516.540042:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774516.540046:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000800:00000200:0.1:1775774516.540051:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:2.0:1775774516.540053:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774516.540060:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774516.540066:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774516.540068:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774516.540070:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774516.540073:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774516.540076:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774516.540080:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774516.540085:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774516.540094:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774516.540101:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774516.540105:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774516.540108:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774516.540113:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774516.540116:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774516.540119:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774516.540125:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775774516.540130:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774516.540178:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774516.540181:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774516.540184:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774516.540187:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774516.540189:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774516.540194:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775774516.540199:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774516.540211:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710270464/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774516.540228:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774516.540237:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774516.540245:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710270464/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774516.540256:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774516.540267:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774516.540278:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774516.540287:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774516.540304:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774516.540317:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11409) 00000800:00000200:0.1:1775774516.540464:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774516.540634:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:2.0:1775774517.691366:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774517.691369:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774517.691373:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774518.715292:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774518.715296:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774518.715299:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774521.659226:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774521.659235:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774521.659238:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774521.659240:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774521.659244:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774521.659250:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11404 00000400:00000200:2.0:1775774521.659259:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774521.659264:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774521.659267:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774521.659270:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774521.659273:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774521.659276:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774521.659279:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774521.659284:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774521.659286:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774521.659291:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774521.659295:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774521.659297:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774521.659301:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774521.659307:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774521.659309:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774521.659310:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774521.659311:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774521.659314:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774521.659319:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774521.659322:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774521.659325:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774521.659327:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774521.659329:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774521.659331:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774521.659333:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774521.659339:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775774521.659342:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774521.659345:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774521.659347:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774521.659349:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774521.659351:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774521.659355:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774521.659361:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775774521.659367:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774521.659374:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710270592/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774521.659382:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774521.659386:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774521.659389:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710270592/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774521.659409:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774521.659412:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774521.659416:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774521.659417:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774521.659419:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774521.659421:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774521.659424:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11404 00000400:00000200:2.0:1775774521.659427:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774521.659430:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774521.659432:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774521.659435:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774521.659437:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774521.659439:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774521.659442:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774521.659444:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774521.659446:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774521.659448:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774521.659451:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774521.659454:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774521.659456:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774521.659460:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774521.659462:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774521.659465:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774521.659469:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774521.659474:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774521.659481:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774521.659486:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774521.659490:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774521.659497:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774521.659502:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774521.659505:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774521.659511:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774521.659517:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774521.659522:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774521.659527:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775774521.659531:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774521.659537:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774521.659540:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774521.659544:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774521.659546:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774521.659550:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774521.659557:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775774521.659564:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774521.659571:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710270720/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774521.659595:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774521.659600:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774521.659605:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710270720/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774521.659617:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774521.659622:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774521.659629:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774521.659631:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774521.659634:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774521.659639:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774521.659645:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11404 00000400:00000200:2.0:1775774521.659653:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774521.659662:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774521.659666:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774521.659672:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774521.659675:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774521.659679:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774521.659682:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774521.659684:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774521.659686:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774521.659689:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774521.659691:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774521.659695:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774521.659697:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774521.659702:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774521.659705:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774521.659710:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774521.659713:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774521.659718:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774521.659726:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774521.659729:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774521.659733:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774521.659736:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774521.659741:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774521.659745:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774521.659748:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774521.659753:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774521.659757:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774521.659763:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775774521.659769:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774521.659774:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774521.659778:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774521.659787:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774521.659791:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774521.659795:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774521.659800:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775774521.659804:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774521.659810:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710270848/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774521.659826:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774521.659829:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774521.659835:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710270848/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774521.659852:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774521.659859:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775774521.659862:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775774521.659865:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775774521.659868:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000100:00000040:2.0:1775774521.659869:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774521.659876:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774521.659886:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11414) 00000800:00000200:1.1:1775774521.660364:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774521.660366:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774521.660671:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774521.660674:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000020:00000001:2.0:1775774523.835426:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774523.835430:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774523.835433:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774526.779836:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774526.779839:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774526.780302:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774526.780305:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774526.780595:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774526.780597:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000020:00000001:2.0:1775774528.955303:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774528.955307:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774528.955310:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.0:1775774530.683245:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774530.683253:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774530.683257:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.1:1775774531.899851:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774531.899856:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775774531.900281:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774531.900283:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775774531.900479:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774531.900481:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000020:00000001:2.0:1775774534.075373:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774534.075377:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774534.075380:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774537.019258:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774537.019263:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774537.019266:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774537.019268:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774537.019271:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774537.019277:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11419 00000400:00000200:2.0:1775774537.019285:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774537.019290:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774537.019292:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774537.019295:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774537.019298:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774537.019299:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774537.019301:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774537.019304:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774537.019305:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774537.019307:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774537.019309:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774537.019311:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774537.019312:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774537.019315:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774537.019316:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774537.019318:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774537.019319:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774537.019321:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774537.019323:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774537.019325:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774537.019328:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774537.019330:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774537.019332:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774537.019333:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774537.019334:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774537.019341:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775774537.019344:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774537.019347:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774537.019348:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774537.019350:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774537.019352:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774537.019355:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774537.019361:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775774537.019366:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774537.019374:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710271744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774537.019382:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774537.019386:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774537.019389:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710271744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774537.019406:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774537.019409:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774537.019411:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774537.019412:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774537.019414:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774537.019421:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774537.019423:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11419 00000400:00000200:2.0:1775774537.019426:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774537.019429:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774537.019431:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774537.019433:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774537.019435:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774537.019437:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774537.019441:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774537.019444:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774537.019446:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774537.019448:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774537.019451:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774537.019452:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774537.019455:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774537.019459:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774537.019463:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774537.019466:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774537.019471:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774537.019475:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774537.019479:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774537.019483:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774537.019488:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774537.019492:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774537.019499:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774537.019504:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774537.019509:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774537.019514:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774537.019517:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774537.019523:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775774537.019527:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774537.019529:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774537.019530:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774537.019531:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774537.019534:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774537.019536:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774537.019541:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775774537.019546:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774537.019552:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710271872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774537.019561:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774537.019563:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774537.019565:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710271872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774537.019577:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774537.019580:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774537.019582:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774537.019584:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774537.019585:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774537.019589:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774537.019593:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11419 00000400:00000200:2.0:1775774537.019600:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774537.019606:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774537.019610:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774537.019616:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774537.019619:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774537.019621:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774537.019624:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774537.019626:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774537.019628:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774537.019631:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774537.019632:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774537.019634:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774537.019638:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774537.019643:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774537.019644:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774537.019645:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774537.019646:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774537.019649:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774537.019651:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774537.019653:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774537.019654:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774537.019657:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774537.019661:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774537.019663:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774537.019677:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774537.019681:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774537.019684:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774537.019687:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775774537.019690:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774537.019697:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774537.019699:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774537.019703:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774537.019706:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774537.019708:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774537.019712:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775774537.019716:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774537.019725:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710272000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774537.019732:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774537.019734:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774537.019742:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710272000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774537.019757:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774537.019761:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774537.019766:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774537.019769:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774537.019771:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774537.019775:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11429) 00000800:00000200:0.1:1775774537.019791:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774537.019794:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775774537.020166:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774537.020168:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775774537.020411:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774537.020413:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000020:00000001:2.0:1775774539.195336:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774539.195340:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774539.195342:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774542.139253:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774542.139261:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774542.139265:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774542.139270:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774542.139276:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774542.139291:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11424 00000400:00000200:2.0:1775774542.139308:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774542.139319:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774542.139324:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774542.139329:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774542.139332:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774542.139335:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774542.139338:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774542.139343:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774542.139345:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774542.139348:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774542.139351:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774542.139352:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774542.139354:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774542.139357:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774542.139358:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774542.139360:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774542.139361:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774542.139363:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774542.139366:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774542.139367:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774542.139369:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774542.139372:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774542.139374:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774542.139376:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774542.139377:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774542.139386:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775774542.139389:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774542.139392:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774542.139393:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774542.139396:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774542.139398:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774542.139401:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774542.139407:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775774542.139412:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774542.139420:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710272128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774542.139427:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774542.139431:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774542.139434:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710272128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774542.139453:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774542.139456:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774542.139458:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774542.139459:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774542.139461:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774542.139463:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774542.139465:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11424 00000400:00000200:2.0:1775774542.139468:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774542.139470:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774542.139472:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774542.139474:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774542.139475:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774542.139477:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774542.139478:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774542.139479:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774542.139480:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774542.139481:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774542.139483:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774542.139484:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774542.139486:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774542.139487:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774542.139489:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774542.139490:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774542.139492:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774542.139493:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774542.139496:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774542.139497:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774542.139499:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774542.139503:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774542.139504:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774542.139506:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774542.139509:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774542.139513:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774542.139516:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774542.139519:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775774542.139522:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774542.139526:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774542.139528:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774542.139532:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774542.139533:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774542.139536:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774542.139541:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775774542.139546:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774542.139552:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710272256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774542.139561:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774542.139563:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774542.139568:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710272256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774542.139602:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774542.139607:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774542.139610:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774542.139612:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774542.139614:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774542.139619:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774542.139623:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11424 00000400:00000200:2.0:1775774542.139626:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774542.139628:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774542.139634:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774542.139638:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774542.139640:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774542.139642:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774542.139648:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774542.139653:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774542.139654:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774542.139656:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774542.139657:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774542.139662:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774542.139664:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774542.139666:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774542.139667:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774542.139671:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774542.139673:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774542.139675:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774542.139677:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774542.139679:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774542.139682:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774542.139686:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774542.139689:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774542.139692:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774542.139695:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774542.139698:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774542.139700:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774542.139704:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775774542.139708:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774542.139712:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774542.139715:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774542.139718:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774542.139721:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774542.139724:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774542.139728:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775774542.139732:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774542.139739:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710272384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774542.139750:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774542.139754:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774542.139760:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710272384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774542.139766:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774542.139771:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774542.139775:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774542.139777:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774542.139782:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774542.139790:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11434) 00000800:00000200:0.1:1775774542.139857:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774542.139860:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775774542.140307:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774542.140310:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775774542.140638:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774542.140641:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000020:00000001:2.0:1775774544.315337:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774544.315342:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774544.315346:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774547.259842:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774547.259848:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775774547.260218:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774547.260220:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775774547.260469:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774547.260471:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000020:00000001:2.0:1775774549.435356:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774549.435361:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774549.435365:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774552.380021:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774552.380025:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775774552.380575:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774552.380579:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775774552.380888:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774552.380890:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000020:00000001:2.0:1775774554.555434:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774554.555439:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774554.555442:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774557.499323:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774557.499329:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774557.499331:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774557.499334:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774557.499338:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774557.499344:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11440 00000400:00000200:2.0:1775774557.499351:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774557.499355:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774557.499358:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774557.499361:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774557.499363:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774557.499365:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774557.499367:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774557.499370:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774557.499372:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774557.499374:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774557.499376:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774557.499378:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774557.499379:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774557.499381:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774557.499382:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774557.499384:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774557.499385:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774557.499387:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774557.499390:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774557.499397:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774557.499400:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774557.499402:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774557.499404:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774557.499406:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774557.499408:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774557.499415:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775774557.499417:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774557.499420:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774557.499422:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774557.499424:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774557.499426:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774557.499430:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774557.499436:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775774557.499441:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774557.499449:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710273280/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774557.499457:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774557.499461:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774557.499465:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710273280/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774557.499484:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774557.499486:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774557.499489:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774557.499490:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774557.499491:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774557.499493:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774557.499494:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11440 00000400:00000200:2.0:1775774557.499497:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774557.499499:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774557.499500:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774557.499502:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774557.499504:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774557.499509:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774557.499512:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774557.499513:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774557.499515:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774557.499516:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774557.499519:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774557.499520:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774557.499522:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774557.499524:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774557.499525:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774557.499527:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774557.499530:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774557.499533:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774557.499537:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774557.499540:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774557.499543:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774557.499547:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774557.499552:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774557.499555:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774557.499559:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774557.499562:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774557.499565:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774557.499569:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775774557.499571:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774557.499574:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774557.499575:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774557.499579:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774557.499581:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774557.499583:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774557.499589:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775774557.499593:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774557.499603:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710273408/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774557.499615:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774557.499618:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774557.499621:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710273408/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774557.499631:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774557.499635:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774557.499637:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774557.499638:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774557.499641:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774557.499645:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774557.499648:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11440 00000400:00000200:2.0:1775774557.499650:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774557.499655:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774557.499656:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774557.499659:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774557.499660:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774557.499661:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774557.499663:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774557.499665:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774557.499666:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774557.499667:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774557.499668:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774557.499673:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774557.499675:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774557.499677:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774557.499678:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774557.499682:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774557.499684:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774557.499688:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774557.499691:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774557.499696:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774557.499698:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774557.499701:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774557.499704:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774557.499708:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774557.499710:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774557.499715:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774557.499718:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774557.499721:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775774557.499725:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774557.499729:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774557.499731:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774557.499733:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774557.499734:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774557.499738:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774557.499741:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775774557.499745:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774557.499752:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710273536/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774557.499758:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774557.499760:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774557.499765:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710273536/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774557.499773:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774557.499777:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774557.499780:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774557.499782:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774557.499784:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774557.499790:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11450) 00000800:00000200:0.1:1775774557.499848:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774557.499851:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775774557.500227:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774557.500229:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775774557.500457:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774557.500460:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000020:00000001:2.0:1775774559.675344:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774559.675348:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774559.675351:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774562.619946:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774562.619956:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774562.620704:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774562.620707:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774562.620979:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774562.620981:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1775774564.795293:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774564.795296:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774564.795299:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774567.739741:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774567.739747:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774567.740099:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774567.740101:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774567.740374:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774567.740376:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.0:1775774568.571238:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774568.571245:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774568.571248:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000020:00000001:2.0:1775774569.915329:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774569.915333:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774569.915336:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774572.859272:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774572.859282:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774572.859289:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774572.859296:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774572.859306:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774572.859316:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11455 00000400:00000200:2.0:1775774572.859329:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774572.859337:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774572.859342:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774572.859346:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774572.859350:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774572.859353:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774572.859354:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774572.859359:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774572.859360:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774572.859363:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774572.859365:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774572.859367:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774572.859369:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774572.859371:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774572.859373:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774572.859374:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774572.859376:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774572.859379:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774572.859381:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774572.859383:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774572.859386:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774572.859388:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774572.859390:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774572.859392:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774572.859394:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774572.859403:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775774572.859405:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774572.859409:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774572.859410:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774572.859413:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774572.859416:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774572.859419:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774572.859427:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775774572.859432:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774572.859442:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710274432/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774572.859451:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774572.859456:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774572.859460:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710274432/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774572.859483:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774572.859486:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774572.859488:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774572.859489:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774572.859491:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774572.859493:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774572.859496:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11455 00000400:00000200:2.0:1775774572.859500:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774572.859503:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774572.859505:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774572.859508:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774572.859509:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774572.859511:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774572.859513:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774572.859515:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774572.859517:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774572.859521:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774572.859523:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774572.859526:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774572.859529:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774572.859537:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774572.859542:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774572.859547:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774572.859552:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774572.859559:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774572.859563:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774572.859567:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774572.859572:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774572.859578:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774572.859581:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774572.859587:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774572.859591:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774572.859596:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774572.859598:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774572.859603:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775774572.859608:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774572.859610:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774572.859613:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774572.859615:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774572.859617:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774572.859620:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774572.859626:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775774572.859629:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774572.859645:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710274560/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774572.859657:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774572.859660:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774572.859673:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710274560/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774572.859688:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774572.859694:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774572.859701:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774572.859704:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774572.859708:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774572.859714:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774572.859720:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11455 00000400:00000200:2.0:1775774572.859725:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774572.859734:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774572.859736:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774572.859739:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774572.859744:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774572.859750:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774572.859752:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774572.859756:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774572.859759:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774572.859761:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774572.859765:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774572.859768:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774572.859771:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774572.859777:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774572.859782:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774572.859789:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774572.859792:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774572.859796:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774572.859804:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774572.859809:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774572.859812:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774572.859814:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774572.859819:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774572.859823:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774572.859826:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774572.859831:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774572.859834:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774572.859841:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775774572.859845:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774572.859851:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774572.859852:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774572.859854:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774572.859855:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774572.859859:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774572.859866:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775774572.859872:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775774572.859877:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00100000:2.0:1775774572.859882:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710274688/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.1:1775774572.859883:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000100:00000001:2.0:1775774572.859900:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774572.859903:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774572.859909:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710274688/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774572.859924:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774572.859927:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774572.859933:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774572.859938:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774572.859942:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774572.859950:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11465) 00000800:00000200:0.1:1775774572.860285:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774572.860287:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775774572.860566:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774572.860568:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000020:00000001:2.0:1775774575.035317:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774575.035321:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774575.035324:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774577.979192:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774577.979201:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774577.979205:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774577.979210:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774577.979216:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774577.979223:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11460 00000400:00000200:2.0:1775774577.979233:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774577.979239:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774577.979241:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774577.979244:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774577.979252:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774577.979254:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774577.979256:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774577.979259:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774577.979261:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774577.979263:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774577.979265:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774577.979267:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774577.979268:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774577.979270:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774577.979272:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774577.979273:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774577.979275:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774577.979277:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774577.979279:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774577.979280:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774577.979283:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774577.979285:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774577.979290:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774577.979291:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774577.979292:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774577.979300:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775774577.979302:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774577.979305:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774577.979307:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774577.979309:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774577.979312:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774577.979315:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774577.979321:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775774577.979326:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774577.979333:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710274816/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774577.979341:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774577.979345:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774577.979348:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710274816/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774577.979371:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774577.979375:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774577.979378:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774577.979380:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774577.979381:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774577.979384:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774577.979387:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11460 00000400:00000200:2.0:1775774577.979392:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774577.979395:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774577.979398:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774577.979402:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774577.979405:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774577.979407:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774577.979410:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774577.979413:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774577.979415:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774577.979417:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774577.979419:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774577.979421:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774577.979423:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774577.979426:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774577.979428:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774577.979430:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774577.979432:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774577.979435:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774577.979439:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774577.979447:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774577.979453:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774577.979460:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774577.979467:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774577.979480:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774577.979489:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774577.979500:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774577.979503:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774577.979508:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775774577.979515:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774577.979528:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774577.979530:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774577.979538:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774577.979545:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774577.979553:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774577.979558:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775774577.979568:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774577.979575:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710274944/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774577.979598:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774577.979600:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774577.979611:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710274944/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774577.979627:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774577.979635:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774577.979639:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774577.979641:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774577.979644:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774577.979647:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774577.979654:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11460 00000400:00000200:2.0:1775774577.979661:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774577.979677:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774577.979685:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774577.979691:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774577.979701:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774577.979707:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774577.979711:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774577.979718:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774577.979720:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774577.979722:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774577.979727:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774577.979736:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774577.979740:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774577.979747:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774577.979756:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774577.979761:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774577.979766:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774577.979774:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774577.979778:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774577.979784:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774577.979789:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774577.979798:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774577.979807:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774577.979812:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774577.979814:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774577.979816:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774577.979817:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774577.979821:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775774577.979828:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774577.979834:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774577.979837:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774577.979843:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774577.979848:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774577.979851:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774577.979855:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775774577.979863:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774577.979873:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710275072/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774577.979892:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774577.979894:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774577.979901:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710275072/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774577.979924:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774577.979927:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774577.979936:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774577.979943:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.1:1775774577.979943:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775774577.979969:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:0.1:1775774577.979970:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000100:00000040:2.0:1775774577.979975:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11470) 00000800:00000200:0.1:1775774577.980380:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774577.980382:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775774577.980576:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774577.980578:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000020:00000001:2.0:1775774580.155364:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774580.155369:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774580.155373:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774583.099249:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774583.099255:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774583.099258:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774583.099263:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774583.099269:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774583.099274:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774583.099278:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774583.099281:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774583.099287:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775774583.099802:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774583.100051:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774583.100221:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:2.0:1775774584.123334:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774584.123341:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774584.123343:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774584.123349:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774584.123352:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774584.123355:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774584.123358:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774584.123359:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774584.123361:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774585.147273:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774585.147278:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774585.147279:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774585.147283:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774585.147286:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774585.147290:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774585.147292:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774585.147294:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774585.147296:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774585.275321:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774585.275326:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774585.275329:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774588.219328:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774588.219336:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774588.219338:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774588.219341:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774588.219345:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774588.219358:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11470 00000400:00000200:2.0:1775774588.219369:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774588.219376:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774588.219379:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774588.219382:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774588.219386:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774588.219389:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774588.219391:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774588.219396:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774588.219398:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774588.219401:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774588.219404:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774588.219406:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774588.219409:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774588.219412:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774588.219414:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774588.219417:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774588.219419:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774588.219422:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774588.219425:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774588.219428:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774588.219431:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774588.219435:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774588.219437:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774588.219440:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774588.219442:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774588.219449:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775774588.219453:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774588.219458:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774588.219460:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774588.219463:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774588.219466:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774588.219470:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774588.219478:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775774588.219484:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774588.219491:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710275584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774588.219504:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774588.219508:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774588.219513:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710275584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774588.219541:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774588.219545:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774588.219549:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774588.219550:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774588.219552:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774588.219556:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774588.219558:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11470 00000400:00000200:2.0:1775774588.219563:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774588.219566:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774588.219569:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774588.219572:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774588.219574:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774588.219577:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774588.219579:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774588.219581:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774588.219583:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774588.219584:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774588.219586:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774588.219588:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774588.219591:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774588.219594:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774588.219596:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774588.219598:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774588.219600:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774588.219604:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774588.219609:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774588.219612:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774588.219616:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774588.219625:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774588.219628:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774588.219633:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774588.219637:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774588.219645:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774588.219651:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774588.219657:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775774588.219661:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774588.219665:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774588.219669:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774588.219672:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774588.219676:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774588.219679:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774588.219687:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775774588.219692:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774588.219700:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710275712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774588.219711:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774588.219714:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774588.219721:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710275712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774588.219739:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774588.219742:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774588.219746:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774588.219750:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774588.219752:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774588.219755:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774588.219760:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11470 00000400:00000200:2.0:1775774588.219770:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774588.219790:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774588.219793:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774588.219799:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774588.219804:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774588.219810:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774588.219814:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774588.219819:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774588.219823:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774588.219826:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774588.219829:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774588.219832:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774588.219836:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774588.219845:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774588.219851:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774588.219856:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774588.219859:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774588.219862:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774588.219869:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774588.219872:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774588.219876:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774588.219882:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774588.219885:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774588.219889:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774588.219894:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774588.219899:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774588.219902:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774588.219908:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775774588.219913:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774588.219921:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774588.219925:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774588.219928:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774588.219930:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774588.219933:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774588.219940:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775774588.219953:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774588.219963:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710275840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774588.219980:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774588.219984:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775774588.219985:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774588.219992:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000100:00000040:2.0:1775774588.219993:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710275840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774588.220017:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774588.220023:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774588.220031:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774588.220035:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774588.220044:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774588.220052:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11480) 00000800:00000200:0.1:1775774588.220496:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774588.220500:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775774588.220816:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774588.220819:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000020:00000001:2.0:1775774590.395310:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774590.395315:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774590.395318:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774593.339254:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774593.339263:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774593.339268:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774593.339273:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774593.339279:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774593.339287:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11475 00000400:00000200:2.0:1775774593.339297:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774593.339304:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774593.339308:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774593.339313:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774593.339316:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774593.339319:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774593.339321:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774593.339325:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774593.339327:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774593.339330:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774593.339333:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774593.339335:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774593.339338:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774593.339341:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774593.339343:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774593.339345:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774593.339347:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774593.339350:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774593.339353:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774593.339356:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774593.339360:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774593.339363:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774593.339371:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774593.339374:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774593.339376:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774593.339385:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775774593.339389:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774593.339392:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774593.339394:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774593.339397:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774593.339400:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774593.339404:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774593.339412:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775774593.339420:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774593.339429:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710275968/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774593.339440:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774593.339445:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774593.339449:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710275968/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774593.339474:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774593.339477:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774593.339481:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774593.339482:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774593.339484:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774593.339487:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774593.339490:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11475 00000400:00000200:2.0:1775774593.339495:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774593.339498:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774593.339501:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774593.339504:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774593.339507:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774593.339511:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774593.339515:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774593.339518:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774593.339520:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774593.339523:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774593.339525:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774593.339530:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774593.339535:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774593.339544:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774593.339548:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774593.339551:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774593.339556:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774593.339561:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774593.339568:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774593.339572:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774593.339576:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774593.339583:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774593.339587:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774593.339590:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774593.339596:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774593.339602:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774593.339606:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774593.339614:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775774593.339618:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774593.339623:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774593.339624:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774593.339626:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774593.339627:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774593.339629:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774593.339636:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775774593.339641:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774593.339651:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710276096/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774593.339674:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774593.339679:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774593.339702:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710276096/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774593.339712:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774593.339715:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774593.339718:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774593.339720:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774593.339722:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774593.339724:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774593.339727:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11475 00000400:00000200:2.0:1775774593.339733:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774593.339737:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774593.339740:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774593.339745:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774593.339747:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774593.339751:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774593.339752:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774593.339754:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774593.339755:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774593.339756:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774593.339759:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774593.339762:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774593.339764:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774593.339766:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774593.339768:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774593.339772:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774593.339776:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774593.339779:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774593.339782:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774593.339786:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774593.339788:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774593.339790:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774593.339794:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774593.339797:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774593.339799:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774593.339803:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774593.339805:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774593.339810:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775774593.339814:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774593.339819:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774593.339822:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774593.339826:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774593.339829:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774593.339832:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774593.339835:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775774593.339839:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774593.339847:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710276224/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774593.339863:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774593.339866:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774593.339871:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710276224/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774593.339884:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774593.339890:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774593.339896:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774593.339903:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.1:1775774593.339909:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775774593.339910:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:0.1:1775774593.339916:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000100:00000040:2.0:1775774593.339919:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11486) 00000800:00000200:0.1:1775774593.340391:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774593.340394:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1775774593.340728:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774593.340730:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000020:00000001:2.0:1775774595.515370:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774595.515375:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774595.515379:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774598.459235:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774598.459245:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774598.459249:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774598.459253:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774598.459259:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774598.459268:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11481 00000400:00000200:2.0:1775774598.459278:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774598.459284:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774598.459287:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774598.459290:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774598.459293:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774598.459295:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774598.459297:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774598.459300:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774598.459301:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774598.459303:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774598.459305:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774598.459307:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774598.459308:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774598.459310:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774598.459311:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774598.459313:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774598.459314:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774598.459316:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774598.459319:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774598.459320:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774598.459323:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774598.459325:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774598.459327:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774598.459329:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774598.459330:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774598.459338:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775774598.459340:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774598.459343:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774598.459344:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774598.459347:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774598.459349:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774598.459353:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774598.459360:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775774598.459367:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774598.459376:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710276352/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774598.459388:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774598.459393:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774598.459398:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710276352/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774598.459422:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774598.459425:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774598.459427:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774598.459428:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774598.459430:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774598.459432:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774598.459435:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11481 00000400:00000200:2.0:1775774598.459439:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774598.459442:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774598.459444:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774598.459448:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774598.459450:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774598.459454:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774598.459456:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774598.459459:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774598.459461:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774598.459464:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774598.459467:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774598.459471:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774598.459476:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774598.459483:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774598.459485:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774598.459488:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774598.459492:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774598.459496:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774598.459500:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774598.459505:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774598.459506:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774598.459511:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774598.459513:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774598.459516:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774598.459520:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774598.459524:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774598.459527:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774598.459532:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775774598.459534:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774598.459538:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774598.459540:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774598.459542:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774598.459543:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774598.459546:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774598.459552:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775774598.459555:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774598.459561:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710276480/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774598.459573:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774598.459576:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774598.459584:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710276480/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774598.459618:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774598.459621:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774598.459625:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774598.459627:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774598.459628:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774598.459633:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774598.459637:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11481 00000400:00000200:2.0:1775774598.459644:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774598.459650:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774598.459654:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774598.459659:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774598.459662:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774598.459667:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774598.459670:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774598.459673:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774598.459676:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774598.459679:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774598.459682:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774598.459686:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774598.459692:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774598.459698:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774598.459700:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774598.459702:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774598.459705:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774598.459709:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774598.459715:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774598.459719:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774598.459721:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774598.459723:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774598.459726:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774598.459730:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774598.459732:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774598.459734:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774598.459736:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774598.459741:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775774598.459743:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774598.459748:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774598.459750:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774598.459753:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774598.459754:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774598.459756:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774598.459760:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775774598.459763:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774598.459770:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710276608/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.1:1775774598.459774:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:2.0:1775774598.459782:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:0.1:1775774598.459783:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000100:00000001:2.0:1775774598.459785:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774598.459792:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710276608/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774598.459808:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774598.459812:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774598.459820:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774598.459825:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774598.459832:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774598.459840:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11491) 00000800:00000200:0.1:1775774598.460094:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774598.460096:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1775774598.460308:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774598.460310:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000020:00000001:2.0:1775774600.635321:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774600.635324:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774600.635326:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774603.580077:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774603.580081:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775774603.580465:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774603.580466:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775774603.580709:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774603.580713:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000020:00000001:2.0:1775774605.755342:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774605.755345:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774605.755348:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774608.699795:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774608.699801:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775774608.700342:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774608.700345:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775774608.700661:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774608.700664:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000020:00000001:2.0:1775774610.875271:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774610.875275:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774610.875278:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774613.819740:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774613.819745:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775774613.820129:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774613.820130:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775774613.820353:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774613.820354:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000020:00000001:2.0:1775774615.995313:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774615.995320:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774615.995325:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774618.940176:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774618.940180:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775774618.940652:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774618.940654:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775774618.940996:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774618.940999:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000020:00000001:2.0:1775774621.115357:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774621.115361:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774621.115364:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774624.059730:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774624.059734:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774624.060037:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774624.060039:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774624.060267:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774624.060269:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000020:00000001:2.0:1775774626.235341:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774626.235345:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774626.235348:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774629.179284:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774629.179292:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774629.179297:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774629.179308:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774629.179319:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774629.179327:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774629.179335:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774629.179339:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774629.179343:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774629.179851:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774629.179855:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774629.180261:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774629.180262:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774629.180443:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774629.180444:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1775774630.203224:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774630.203228:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774630.203230:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774630.203234:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774630.203237:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774630.203240:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774630.203242:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774630.203244:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774630.203246:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774631.227248:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774631.227253:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774631.227254:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774631.227259:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774631.227262:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774631.227265:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774631.227267:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774631.227269:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774631.227271:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774631.355327:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774631.355330:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774631.355333:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774632.251235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774632.251243:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774632.251245:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774632.251252:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774632.251256:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774632.251260:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774632.251263:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774632.251265:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774632.251268:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774633.275246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774633.275251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774633.275253:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774633.275257:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774633.275260:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774633.275264:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774633.275266:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774633.275268:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774633.275270:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774634.299202:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774634.299208:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774634.299210:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774634.299214:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774634.299218:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774634.299221:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774634.299223:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774634.299225:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774634.299228:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775774634.299885:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774634.299889:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1775774634.300353:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774634.300356:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1775774634.300638:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774634.300640:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 11 secs later) 00000100:00000001:2.0:1775774635.323273:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774635.323280:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774635.323282:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774635.323288:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774635.323291:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774635.323295:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774635.323298:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774635.323300:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774635.323303:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774636.347193:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774636.347198:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774636.347200:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774636.347204:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774636.347208:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774636.347211:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774636.347213:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774636.347216:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774636.347217:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774636.475336:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774636.475340:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774636.475343:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774639.419799:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774639.419803:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775774639.420298:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774639.420300:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775774639.420611:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774639.420614:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000020:00000001:2.0:1775774641.595336:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774641.595345:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774641.595348:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774644.539910:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774644.539920:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775774644.540402:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774644.540404:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775774644.540629:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774644.540631:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.0:1775774645.371256:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774645.371263:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774645.371268:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000020:00000001:2.0:1775774646.715359:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774646.715364:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774646.715367:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774649.660078:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774649.660353:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774649.660504:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:2.0:1775774651.835317:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774651.835321:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774651.835325:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775774652.731523:0:2401:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:2.0:1775774652.731577:0:2401:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1775774652.731591:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774652.731595:0:2401:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:2.0:1775774652.731602:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774652.731607:0:2401:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:2.0:1775774652.731614:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775774652.731619:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:2.0:1775774652.731626:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775774652.731632:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775774652.731640:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0440 00000400:00000010:2.0:1775774652.731645:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0440. 00000100:00000001:2.0:1775774652.731664:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775774652.731675:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c71500 x1862022710280192/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775774674 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774652.731717:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775774652.731722:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71500 x1862022710280192/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775774674 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774652.731731:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774652.731733:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775774652.731747:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775774652.731750:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:2.0:1775774652.731754:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775774652.731758:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775774652.731763:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0220 00000400:00000010:2.0:1775774652.731766:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0220. 00000100:00000001:2.0:1775774652.731772:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775774652.731779:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c70700 x1862022710280320/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775774674 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774652.731792:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775774652.731799:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70700 x1862022710280320/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775774674 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774652.731812:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774652.731814:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775774652.731825:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775774652.731831:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c600. 00000400:00000200:2.0:1775774652.731834:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775774652.731841:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775774652.731847:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0ee0 00000400:00000010:2.0:1775774652.731852:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0ee0. 00000100:00000001:2.0:1775774652.731857:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775774652.731866:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c70a80 x1862022710280448/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775774674 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774652.731883:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775774652.731890:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70a80 x1862022710280448/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775774674 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774652.731905:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774652.731908:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000200:0.1:1775774654.779967:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774654.779974:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775774654.780453:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774654.780455:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775774654.780800:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774654.780802:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000020:00000001:2.0:1775774656.955409:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774656.955416:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774656.955422:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774659.899729:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774659.899733:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774659.900128:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774659.900130:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774659.900359:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774659.900361:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000020:00000001:2.0:1775774662.075408:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774662.075412:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774662.075416:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774665.020055:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774665.020060:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774665.020571:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774665.020574:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774665.020786:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774665.020788:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000020:00000001:2.0:1775774667.195371:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774667.195376:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774667.195380:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774670.139307:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774670.139314:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774670.139318:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774670.139325:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774670.139330:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774670.139337:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774670.139366:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774670.139371:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774670.139377:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775774670.139961:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774670.139969:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775774670.140649:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774670.140652:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775774670.140841:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774670.140843:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000100:00000001:2.0:1775774671.163205:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774671.163211:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774671.163212:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774671.163217:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774671.163220:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774671.163223:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774671.163226:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774671.163228:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774671.163229:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774672.187225:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774672.187233:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774672.187235:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774672.187240:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774672.187244:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774672.187247:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774672.187250:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774672.187254:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774672.187256:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774672.315337:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774672.315340:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774672.315342:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774673.211214:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774673.211222:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774673.211224:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774673.211231:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774673.211235:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774673.211239:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774673.211242:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774673.211245:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774673.211248:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774674.235225:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774674.235232:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774674.235233:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774674.235237:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774674.235240:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774674.235243:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774674.235245:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774674.235247:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774674.235249:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774675.259226:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774675.259234:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774675.259236:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774675.259243:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774675.259249:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774675.259268:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774675.259274:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774675.259279:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774675.259285:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774675.259797:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774675.259801:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775774675.260285:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774675.260287:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775774675.260624:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774675.260627:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000020:00000001:2.0:1775774677.435378:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774677.435382:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774677.435386:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774680.379306:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774680.379312:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774680.379315:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774680.379321:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774680.379327:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774680.379332:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774680.379335:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774680.379338:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774680.379341:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774680.379905:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774680.379911:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775774680.380282:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774680.380284:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775774680.380557:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774680.380558:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000100:00000001:2.0:1775774681.403267:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774681.403277:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774681.403279:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774681.403288:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774681.403294:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774681.403298:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774681.403302:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774681.403305:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774681.403308:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774682.427276:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774682.427284:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774682.427286:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774682.427292:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774682.427295:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774682.427299:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774682.427302:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774682.427305:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774682.427307:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774682.555351:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774682.555357:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774682.555361:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774683.451268:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774683.451279:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774683.451281:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774683.451288:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774683.451292:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774683.451296:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774683.451300:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774683.451302:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774683.451306:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775774684.283324:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774684.283331:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774684.283337:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000100:00000001:2.0:1775774684.475245:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774684.475252:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774684.475263:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774684.475269:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774684.475273:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774684.475277:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774684.475279:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774684.475282:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774684.475285:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774685.499199:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774685.499205:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774685.499207:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774685.499212:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774685.499216:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774685.499219:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774685.499221:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774685.499224:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774685.499225:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774685.499763:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774685.499766:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775774685.500091:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774685.500093:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775774685.500308:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774685.500309:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000100:00000001:2.0:1775774686.523220:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774686.523226:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774686.523227:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774686.523231:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774686.523233:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774686.523236:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774686.523239:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774686.523240:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774686.523242:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774687.547212:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774687.547217:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774687.547219:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774687.547223:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774687.547226:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774687.547228:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774687.547230:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774687.547232:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774687.547234:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774687.675349:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774687.675352:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774687.675355:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774690.619864:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774690.619869:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774690.620257:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774690.620259:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774690.620554:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774690.620557:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000020:00000001:2.0:1775774692.795321:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774692.795325:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774692.795328:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774695.740004:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774695.740014:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774695.740497:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774695.740499:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774695.740686:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774695.740688:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1775774697.915317:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774697.915322:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774697.915327:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774700.860045:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774700.860048:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774700.860461:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774700.860464:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774700.860699:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774700.860701:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000020:00000001:2.0:1775774703.035331:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774703.035335:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774703.035339:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774705.979683:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774705.979687:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774705.980071:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774705.980073:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774705.980278:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774705.980280:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000020:00000001:2.0:1775774708.155283:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774708.155288:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774708.155291:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774711.099754:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774711.099757:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774711.100117:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774711.100118:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774711.100324:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774711.100326:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000020:00000001:2.0:1775774713.275293:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774713.275298:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774713.275302:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774716.219748:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774716.219970:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774716.220082:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:2.0:1775774718.395356:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774718.395360:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774718.395364:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774721.339306:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774721.339312:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774721.339315:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774721.339320:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774721.339324:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774721.339328:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774721.339331:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774721.339333:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774721.339337:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775774721.340019:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774721.340025:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775774721.340547:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774721.340549:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775774721.340856:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774721.340859:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000100:00000001:2.0:1775774722.363225:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774722.363231:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774722.363233:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774722.363237:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774722.363240:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774722.363243:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774722.363245:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774722.363247:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774722.363249:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775774723.195245:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774723.195253:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774723.195258:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000100:00000001:2.0:1775774723.387269:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774723.387276:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774723.387278:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774723.387283:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774723.387287:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774723.387291:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774723.387293:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774723.387307:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774723.387310:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774723.515281:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774723.515284:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774723.515287:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774724.411262:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774724.411269:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774724.411272:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774724.411278:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774724.411281:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774724.411285:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774724.411295:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774724.411297:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774724.411299:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774725.435210:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774725.435216:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774725.435217:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774725.435222:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774725.435225:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774725.435227:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774725.435229:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774725.435231:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774725.435233:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774726.459266:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774726.459275:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774726.459279:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774726.459286:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774726.459293:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774726.459299:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774726.459305:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774726.459309:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774726.459315:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775774726.460082:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774726.460092:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775774726.460635:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774726.460637:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775774726.461046:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774726.461050:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000100:00000001:2.0:1775774727.483241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774727.483249:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774727.483251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774727.483256:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774727.483260:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774727.483264:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774727.483267:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774727.483269:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774727.483272:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774728.507264:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774728.507289:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774728.507291:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774728.507298:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774728.507301:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774728.507305:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774728.507308:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774728.507311:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774728.507313:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774728.635369:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774728.635373:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774728.635377:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774731.579233:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774731.579238:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774731.579242:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774731.579246:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774731.579251:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774731.579254:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774731.579259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774731.579262:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774731.579265:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775774731.579971:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774731.579974:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775774731.580387:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774731.580389:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775774731.580630:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774731.580632:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1775774732.603243:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774732.603250:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774732.603251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774732.603280:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774732.603283:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774732.603286:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774732.603288:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774732.603290:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774732.603292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774733.627216:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774733.627221:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774733.627223:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774733.627227:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774733.627230:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774733.627233:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774733.627250:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774733.627269:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774733.627272:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774733.755309:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774733.755312:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774733.755315:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774734.651291:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774734.651299:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774734.651302:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774734.651309:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774734.651314:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774734.651319:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774734.651323:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774734.651326:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774734.651329:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774735.675241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774735.675248:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774735.675250:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774735.675255:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774735.675259:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774735.675263:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774735.675266:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774735.675268:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774735.675271:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774736.699294:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774736.699305:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774736.699308:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774736.699315:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774736.699321:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774736.699333:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774736.699342:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774736.699347:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774736.699353:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775774736.699895:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774736.699900:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775774736.700376:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774736.700378:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775774736.700584:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774736.700587:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000100:00000001:2.0:1775774737.723270:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774737.723277:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774737.723279:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774737.723285:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774737.723289:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774737.723294:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774737.723297:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774737.723299:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774737.723301:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774738.747285:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774738.747292:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774738.747293:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774738.747299:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774738.747303:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774738.747307:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774738.747309:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774738.747312:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774738.747314:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774738.875335:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774738.875340:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774738.875344:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774741.819946:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774741.819951:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775774741.820493:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774741.820495:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775774741.820819:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774741.820822:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000020:00000001:2.0:1775774743.995306:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774743.995311:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774743.995315:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774746.939989:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774746.940001:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775774746.940686:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774746.940691:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775774746.941126:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774746.941129:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000020:00000001:2.0:1775774749.115319:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774749.115323:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774749.115326:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774752.059796:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774752.059800:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775774752.060285:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774752.060288:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775774752.060614:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774752.060617:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000020:00000001:2.0:1775774754.235313:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774754.235317:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774754.235321:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774757.179892:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774757.179900:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 23 secs later) 00000800:00000200:1.1:1775774757.180323:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774757.180325:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 23 secs later) 00000800:00000200:1.1:1775774757.180620:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774757.180622:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 23 secs later) 00000020:00000001:2.0:1775774759.355381:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774759.355386:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774759.355390:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.0:1775774761.083271:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774761.083281:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774761.083291:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:1.1:1775774762.299705:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774762.299709:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774762.300109:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774762.300111:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774762.300328:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774762.300330:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1775774764.475397:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774764.475407:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774764.475414:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774767.419985:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774767.419989:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774767.420478:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774767.420481:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774767.420840:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774767.420842:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000020:00000001:2.0:1775774769.595354:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774769.595357:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774769.595360:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774772.539858:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774772.539864:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774772.540261:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774772.540263:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774772.540493:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774772.540495:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000020:00000001:2.0:1775774774.715428:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774774.715433:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774774.715438:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774777.660318:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774777.660324:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775774777.660975:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774777.660978:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775774777.661315:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774777.661318:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000020:00000001:2.0:1775774779.835332:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774779.835336:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774779.835338:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774782.779726:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774782.779939:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774782.780028:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:2.0:1775774784.955397:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774784.955402:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774784.955405:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774787.899873:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774787.899878:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775774787.900470:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774787.900472:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775774787.900809:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774787.900811:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000020:00000001:2.0:1775774790.075296:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774790.075300:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774790.075302:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774793.019770:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774793.019775:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775774793.020188:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774793.020190:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775774793.020402:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774793.020405:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000020:00000001:2.0:1775774795.195322:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774795.195325:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774795.195327:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774798.139684:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774798.139689:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775774798.140073:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774798.140075:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775774798.140318:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774798.140319:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.0:1775774799.995266:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774799.995276:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774799.995283:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000020:00000001:2.0:1775774800.315299:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774800.315304:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774800.315308:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774803.259280:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774803.259289:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774803.259292:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774803.259295:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774803.259299:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774803.259307:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11685 00000400:00000200:2.0:1775774803.259321:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774803.259326:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774803.259329:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774803.259331:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774803.259333:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774803.259335:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774803.259337:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774803.259340:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774803.259341:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774803.259343:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774803.259345:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774803.259347:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774803.259349:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774803.259351:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774803.259352:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774803.259353:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774803.259355:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774803.259357:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774803.259359:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774803.259360:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774803.259363:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774803.259365:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774803.259367:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774803.259369:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774803.259370:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774803.259377:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775774803.259380:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774803.259383:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774803.259384:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774803.259387:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774803.259389:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774803.259393:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774803.259399:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775774803.259404:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774803.259411:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710291712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774803.259419:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774803.259422:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774803.259425:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710291712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774803.259441:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774803.259444:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774803.259446:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774803.259447:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774803.259449:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774803.259451:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774803.259453:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11685 00000400:00000200:2.0:1775774803.259456:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774803.259459:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774803.259461:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774803.259465:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774803.259468:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774803.259472:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774803.259474:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774803.259476:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774803.259478:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774803.259481:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774803.259484:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774803.259489:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774803.259492:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774803.259497:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774803.259499:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774803.259502:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774803.259507:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774803.259509:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774803.259515:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774803.259521:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774803.259524:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774803.259529:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774803.259534:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774803.259536:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774803.259541:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774803.259546:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774803.259551:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774803.259558:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775774803.259564:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774803.259569:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774803.259572:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774803.259577:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774803.259581:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774803.259585:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774803.259594:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775774803.259600:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774803.259612:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710291840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774803.259628:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774803.259633:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774803.259639:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710291840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774803.259678:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774803.259681:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774803.259684:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774803.259686:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774803.259689:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774803.259693:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774803.259698:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11685 00000400:00000200:2.0:1775774803.259707:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774803.259714:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774803.259717:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774803.259720:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774803.259726:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774803.259728:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774803.259731:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774803.259735:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774803.259737:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774803.259741:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774803.259745:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774803.259750:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:0.1:1775774803.259755:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:2.0:1775774803.259756:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:0.1:1775774803.259761:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000100:00000040:2.0:1775774803.259764:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774803.259771:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774803.259777:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774803.259780:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774803.259785:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774803.259794:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774803.259809:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774803.259815:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774803.259822:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774803.259828:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774803.259832:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774803.259837:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774803.259841:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774803.259846:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774803.259853:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775774803.259857:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774803.259861:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774803.259862:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774803.259863:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774803.259867:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774803.259869:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774803.259873:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775774803.259878:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774803.259886:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710291968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774803.259897:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774803.259900:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774803.259909:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710291968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774803.259921:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774803.259924:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774803.259930:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774803.259935:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774803.259940:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774803.259948:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11695) 00000800:00000200:0.1:1775774803.260198:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774803.260200:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775774803.260448:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774803.260450:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000020:00000001:2.0:1775774805.435284:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774805.435289:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774805.435293:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774808.379967:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774808.379972:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775774808.380474:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774808.380477:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775774808.380837:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774808.380840:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000020:00000001:2.0:1775774810.555342:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774810.555346:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774810.555349:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774813.499726:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774813.499732:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775774813.500238:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774813.500240:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775774813.500493:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774813.500495:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000020:00000001:2.0:1775774815.675390:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774815.675396:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774815.675402:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774818.619286:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774818.619292:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774818.619294:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774818.619297:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774818.619300:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774818.619307:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11701 00000400:00000200:2.0:1775774818.619315:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774818.619320:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774818.619323:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774818.619325:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774818.619328:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774818.619329:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774818.619331:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774818.619334:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774818.619335:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774818.619337:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774818.619340:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774818.619341:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774818.619343:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774818.619345:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774818.619346:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774818.619347:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774818.619349:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774818.619351:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774818.619354:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774818.619356:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774818.619358:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774818.619361:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774818.619363:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774818.619364:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774818.619366:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774818.619373:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775774818.619376:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774818.619379:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774818.619380:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774818.619382:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774818.619384:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774818.619388:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774818.619394:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775774818.619399:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774818.619406:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710292864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774818.619414:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774818.619418:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774818.619421:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710292864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774818.619439:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774818.619441:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774818.619444:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774818.619445:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774818.619447:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774818.619448:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774818.619451:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11701 00000400:00000200:2.0:1775774818.619454:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774818.619456:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774818.619458:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774818.619462:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774818.619464:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774818.619467:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774818.619474:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774818.619477:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774818.619479:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774818.619482:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774818.619485:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774818.619489:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774818.619492:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774818.619497:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774818.619500:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774818.619504:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774818.619509:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774818.619512:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774818.619518:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774818.619526:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774818.619529:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774818.619535:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774818.619537:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774818.619539:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774818.619544:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774818.619547:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774818.619551:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774818.619563:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775774818.619568:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774818.619573:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774818.619576:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774818.619578:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774818.619579:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774818.619581:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774818.619587:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775774818.619593:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774818.619600:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710292992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774818.619612:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774818.619615:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774818.619619:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710292992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774818.619631:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774818.619635:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774818.619640:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774818.619642:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774818.619645:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774818.619648:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774818.619653:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11701 00000400:00000200:2.0:1775774818.619660:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774818.619666:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774818.619670:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774818.619676:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774818.619681:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774818.619688:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774818.619693:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774818.619698:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774818.619700:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774818.619705:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774818.619707:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774818.619711:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774818.619716:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:0.1:1775774818.619722:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775774818.619724:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000800:00000200:0.1:1775774818.619726:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000100:00000001:2.0:1775774818.619729:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774818.619735:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774818.619740:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774818.619744:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774818.619775:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774818.619778:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774818.619782:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774818.619785:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774818.619788:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774818.619790:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774818.619793:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774818.619796:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774818.619799:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774818.619804:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775774818.619808:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774818.619811:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774818.619813:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774818.619815:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774818.619818:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774818.619820:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774818.619824:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775774818.619828:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774818.619837:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710293120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774818.619849:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774818.619850:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774818.619853:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710293120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774818.619863:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774818.619866:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774818.619870:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774818.619874:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774818.619878:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774818.619884:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11711) 00000800:00000200:0.1:1775774818.620160:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774818.620163:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775774818.620437:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774818.620439:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000020:00000001:2.0:1775774820.795303:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774820.795306:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774820.795309:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774823.739963:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774823.739970:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774823.740398:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774823.740401:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774823.740706:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774823.740708:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000020:00000001:2.0:1775774825.915338:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774825.915342:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774825.915345:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774828.859792:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774828.859798:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774828.860101:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774828.860103:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774828.860340:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774828.860341:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1775774831.035304:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774831.035309:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774831.035313:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774832.955315:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774832.955321:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774832.955322:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774832.955327:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774832.955331:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774832.955334:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774832.955336:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774832.955338:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774832.955340:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774833.979233:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774833.979241:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774833.979244:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774833.979248:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774833.979252:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774833.979259:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11716 00000400:00000200:2.0:1775774833.979270:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774833.979279:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774833.979282:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774833.979285:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774833.979288:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774833.979290:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774833.979292:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774833.979295:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774833.979297:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774833.979299:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774833.979301:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774833.979303:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774833.979304:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774833.979307:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774833.979309:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774833.979310:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774833.979312:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774833.979314:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774833.979316:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774833.979318:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774833.979320:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774833.979324:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774833.979326:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774833.979329:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774833.979331:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774833.979340:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775774833.979343:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774833.979347:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774833.979349:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774833.979352:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774833.979355:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774833.979359:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774833.979367:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775774833.979374:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774833.979382:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710294016/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774833.979390:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774833.979394:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774833.979396:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710294016/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774833.979414:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774833.979417:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774833.979419:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774833.979420:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774833.979421:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774833.979424:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774833.979425:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11716 00000400:00000200:2.0:1775774833.979428:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774833.979431:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774833.979433:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774833.979436:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774833.979438:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774833.979440:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774833.979442:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774833.979443:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774833.979444:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774833.979446:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774833.979449:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774833.979451:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774833.979454:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774833.979458:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774833.979460:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774833.979463:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774833.979467:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774833.979471:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774833.979476:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774833.979478:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774833.979480:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774833.979505:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774833.979509:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774833.979512:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774833.979516:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774833.979520:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774833.979524:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774833.979530:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775774833.979533:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774833.979536:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774833.979539:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774833.979541:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774833.979543:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774833.979545:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774833.979549:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775774833.979554:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774833.979561:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710294144/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774833.979571:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774833.979573:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774833.979577:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710294144/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774833.979590:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774833.979595:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774833.979598:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774833.979600:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774833.979601:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774833.979606:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774833.979610:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11716 00000400:00000200:2.0:1775774833.979618:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774833.979623:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774833.979625:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774833.979630:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774833.979631:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774833.979636:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774833.979640:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774833.979642:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774833.979644:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774833.979647:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774833.979649:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774833.979650:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774833.979656:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774833.979658:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774833.979660:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774833.979664:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774833.979668:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774833.979670:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774833.979671:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774833.979673:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774833.979677:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774833.979679:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774833.979683:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774833.979685:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774833.979688:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774833.979693:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774833.979695:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774833.979701:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775774833.979706:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774833.979710:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774833.979714:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774833.979718:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774833.979721:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774833.979725:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774833.979730:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775774833.979737:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774833.979745:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710294272/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774833.979762:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774833.979766:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774833.979770:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710294272/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774833.979779:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774833.979783:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774833.979787:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774833.979791:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774833.979796:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775774833.979797:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774833.979802:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000100:00000040:2.0:1775774833.979803:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11726) 00000800:00000200:1.1:1775774833.980087:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774833.980089:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774833.980313:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774833.980315:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000020:00000001:2.0:1775774836.155390:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774836.155394:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774836.155397:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.0:1775774837.883222:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774837.883233:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774837.883238:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000100:00000001:2.0:1775774839.099288:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774839.099295:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774839.099299:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774839.099310:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774839.099315:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774839.099324:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774839.099330:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774839.099334:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774839.099343:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774839.099815:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774839.099821:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774839.100219:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774839.100221:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774839.100459:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774839.100460:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000100:00000001:2.0:1775774840.123241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774840.123246:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774840.123248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774840.123253:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774840.123256:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774840.123259:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774840.123261:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774840.123263:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774840.123266:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774841.147234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774841.147240:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774841.147242:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774841.147246:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774841.147249:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774841.147252:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774841.147255:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774841.147257:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774841.147259:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774841.275394:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774841.275399:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774841.275403:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774844.220100:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774844.220105:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774844.220697:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774844.220699:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774844.220923:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774844.220925:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000020:00000001:2.0:1775774846.395263:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774846.395277:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774846.395281:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774849.339979:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774849.340416:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774849.340582:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:2.0:1775774851.515303:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774851.515308:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774851.515312:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775774854.459288:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774854.459294:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775774854.459296:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774854.459299:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774854.459302:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774854.459309:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11737 00000400:00000200:2.0:1775774854.459318:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774854.459325:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774854.459329:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774854.459332:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774854.459336:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774854.459339:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774854.459342:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774854.459345:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774854.459348:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775774854.459351:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774854.459353:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775774854.459356:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775774854.459359:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775774854.459362:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774854.459364:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774854.459366:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774854.459369:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774854.459372:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775774854.459375:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774854.459378:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774854.459381:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774854.459384:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774854.459388:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774854.459390:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775774854.459392:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774854.459397:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775774854.459401:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775774854.459405:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774854.459407:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774854.459409:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774854.459412:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774854.459417:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774854.459425:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775774854.459432:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774854.459441:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710295552/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774854.459458:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774854.459462:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774854.459466:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710295552/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774854.459492:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774854.459495:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774854.459498:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775774854.459500:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774854.459501:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774854.459504:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774854.459506:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11737 00000400:00000200:2.0:1775774854.459510:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774854.459514:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774854.459516:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774854.459519:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774854.459520:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774854.459529:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774854.459531:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774854.459534:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774854.459535:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775774854.459537:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774854.459539:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775774854.459540:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775774854.459542:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775774854.459545:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774854.459547:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774854.459549:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774854.459551:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774854.459554:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775774854.459557:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774854.459560:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774854.459561:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774854.459568:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774854.459570:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774854.459572:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774854.459576:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774854.459579:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774854.459581:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774854.459585:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775774854.459588:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774854.459590:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774854.459592:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774854.459594:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774854.459595:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774854.459597:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774854.459602:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775774854.459605:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774854.459609:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710295680/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774854.459615:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774854.459618:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774854.459622:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710295680/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774854.459630:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775774854.459633:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775774854.459636:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775774854.459637:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775774854.459639:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775774854.459642:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775774854.459646:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11737 00000400:00000200:2.0:1775774854.459651:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774854.459655:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775774854.459657:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775774854.459660:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774854.459663:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774854.459665:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774854.459667:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775774854.459670:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775774854.459672:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775774854.459674:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775774854.459675:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775774854.459683:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775774854.459687:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775774854.459692:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774854.459694:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775774854.459699:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775774854.459702:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775774854.459708:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775774854.459715:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775774854.459722:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775774854.459726:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775774854.459731:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775774854.459735:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774854.459738:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775774854.459742:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775774854.459746:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774854.459750:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775774854.459755:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775774854.459759:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775774854.459761:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775774854.459764:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774854.459765:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775774854.459767:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775774854.459770:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775774854.459776:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775774854.459781:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774854.459789:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710295808/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775774854.459799:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775774854.459804:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774854.459812:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710295808/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774854.459828:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775774854.459832:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774854.459841:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774854.459845:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775774854.459850:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775774854.459860:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11747) 00000800:00000200:0.1:1775774854.459984:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774854.459988:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775774854.460396:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774854.460399:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775774854.460829:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774854.460832:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000020:00000001:2.0:1775774856.635294:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774856.635298:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774856.635301:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774859.579986:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774859.579993:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775774859.580448:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774859.580450:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775774859.580666:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774859.580668:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000020:00000001:2.0:1775774861.755365:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774861.755371:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774861.755375:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774864.699910:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774864.699919:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774864.700493:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774864.700495:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775774864.700681:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774864.700683:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000020:00000001:2.0:1775774866.875402:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774866.875407:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774866.875410:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774869.819749:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774869.819754:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774869.820206:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774869.820207:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775774869.820414:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774869.820417:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000020:00000001:2.0:1775774871.995330:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774871.995335:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774871.995339:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774874.939791:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774874.939795:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775774874.940261:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774874.940264:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775774874.940589:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774874.940592:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.0:1775774875.771209:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774875.771215:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774875.771220:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000020:00000001:2.0:1775774877.115378:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774877.115383:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774877.115386:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774880.059915:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774880.059921:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775774880.060272:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774880.060273:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775774880.060530:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774880.060532:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000020:00000001:2.0:1775774882.235282:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774882.235286:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774882.235289:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774885.180051:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774885.180063:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775774885.180618:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774885.180621:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775774885.180953:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774885.180956:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000100:00000001:2.0:1775774886.203361:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774886.203369:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774886.203371:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774886.203378:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774886.203384:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774886.203404:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774886.203409:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774886.203413:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774886.203417:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774886.331327:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774886.331331:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774886.331337:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774887.227209:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774887.227215:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774887.227216:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774887.227220:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774887.227224:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774887.227227:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774887.227229:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774887.227232:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774887.227234:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774887.355336:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774887.355341:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774887.355345:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774888.251236:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774888.251242:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774888.251244:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774888.251248:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774888.251252:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774888.251255:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774888.251257:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774888.251259:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774888.251260:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774889.275218:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774889.275225:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774889.275227:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774889.275232:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774889.275236:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774889.275240:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774889.275243:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774889.275245:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774889.275248:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774890.299245:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774890.299251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774890.299253:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774890.299259:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774890.299264:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774890.299268:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774890.299272:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774890.299274:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774890.299278:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774890.299815:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774890.299820:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774890.300163:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774890.300165:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774890.300349:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774890.300351:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1775774891.323189:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774891.323195:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774891.323197:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774891.323201:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774891.323203:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774891.323206:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774891.323208:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774891.323210:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774891.323212:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774891.451339:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774891.451342:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774891.451345:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774892.347203:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774892.347208:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774892.347210:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774892.347213:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774892.347217:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774892.347220:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774892.347222:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774892.347223:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774892.347225:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774892.475299:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774892.475303:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774892.475306:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774895.419697:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774895.419703:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774895.420079:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774895.420081:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774895.420284:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774895.420285:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1775774896.571381:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774896.571387:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774896.571422:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774897.595341:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774897.595346:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774897.595351:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774900.539734:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774900.539740:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774900.540080:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774900.540082:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774900.540283:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774900.540285:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000020:00000001:2.0:1775774901.691380:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774901.691386:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774901.691390:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774902.715314:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774902.715319:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774902.715323:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774905.659746:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774905.659752:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774905.660170:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774905.660172:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775774905.660421:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774905.660423:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000020:00000001:2.0:1775774906.811277:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774906.811281:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774906.811284:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774907.835286:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774907.835290:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774907.835294:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774910.780198:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774910.780202:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774910.780904:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774910.780906:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775774910.781233:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774910.781235:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000020:00000001:2.0:1775774911.931337:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774911.931341:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774911.931344:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774912.955399:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774912.955403:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774912.955407:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.0:1775774914.683264:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774914.683292:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774914.683298:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:1.1:1775774915.899988:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774915.900340:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774915.900499:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:2.0:1775774917.051322:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774917.051326:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774917.051329:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774918.075444:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774918.075450:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774918.075456:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775774918.971539:0:2401:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:2.0:1775774918.971593:0:2401:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1775774918.971608:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774918.971613:0:2401:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:2.0:1775774918.971633:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775774918.971638:0:2401:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:2.0:1775774918.971645:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775774918.971650:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230000. 00000400:00000200:2.0:1775774918.971657:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775774918.971663:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775774918.971671:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f33b8 00000400:00000010:2.0:1775774918.971674:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f33b8. 00000100:00000001:2.0:1775774918.971683:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775774918.971694:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c71c00 x1862022710300160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775774940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774918.971733:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775774918.971738:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71c00 x1862022710300160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775774940 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774918.971746:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774918.971749:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775774918.971759:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775774918.971761:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230c00. 00000400:00000200:2.0:1775774918.971765:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775774918.971769:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775774918.971772:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:2.0:1775774918.971775:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:2.0:1775774918.971778:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775774918.971784:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c70000 x1862022710300288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775774940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774918.971792:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775774918.971796:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70000 x1862022710300288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775774940 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774918.971803:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774918.971805:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775774918.971809:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775774918.971812:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:2.0:1775774918.971824:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775774918.971828:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775774918.971831:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:2.0:1775774918.971834:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:2.0:1775774918.971837:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775774918.971842:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c70380 x1862022710300416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775774940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774918.971850:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775774918.971853:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70380 x1862022710300416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775774940 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775774918.971861:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774918.971863:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000200:1.1:1775774921.019838:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774921.019844:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774921.020302:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774921.020304:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774921.020562:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774921.020564:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000020:00000001:2.0:1775774922.171311:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774922.171315:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774922.171318:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774923.195424:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774923.195431:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774923.195435:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774926.139238:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774926.139244:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774926.139247:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774926.139255:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774926.139260:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774926.139265:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774926.139270:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774926.139275:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774926.139281:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775774926.139665:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774926.139671:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774926.140008:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774926.140009:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775774926.140206:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774926.140208:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000100:00000001:2.0:1775774927.163259:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774927.163265:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774927.163267:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774927.163271:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774927.163275:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774927.163279:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774927.163281:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774927.163283:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774927.163285:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774927.291818:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774927.291821:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774927.291824:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774928.187211:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774928.187217:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774928.187218:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774928.187222:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774928.187225:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774928.187228:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774928.187230:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774928.187232:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774928.187234:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774928.315341:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774928.315346:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774928.315349:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774929.211284:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774929.211293:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774929.211295:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774929.211302:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774929.211316:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774929.211321:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774929.211324:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774929.211327:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774929.211330:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774930.235195:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774930.235201:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774930.235203:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774930.235207:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774930.235214:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774930.235217:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774930.235218:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774930.235220:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774930.235222:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775774931.259259:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774931.259266:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774931.259273:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774931.259280:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774931.259286:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774931.259291:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774931.259296:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774931.259301:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774931.259308:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775774931.259821:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774931.259825:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775774931.260216:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774931.260218:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775774931.260386:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774931.260387:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1775774932.283284:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774932.283293:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774932.283295:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774932.283301:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774932.283305:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774932.283310:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774932.283313:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774932.283316:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774932.283319:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774932.411295:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774932.411298:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774932.411300:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774933.307243:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774933.307250:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774933.307251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774933.307255:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774933.307258:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774933.307260:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774933.307262:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774933.307264:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774933.307266:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775774933.435275:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774933.435278:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774933.435281:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774936.379707:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774936.379710:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775774936.379970:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774936.379972:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775774936.380131:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774936.380132:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000020:00000001:2.0:1775774937.531332:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774937.531336:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774937.531344:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774938.555303:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774938.555308:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774938.555311:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774941.499724:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774941.499730:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775774941.500132:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774941.500155:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775774941.500422:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774941.500424:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000020:00000001:2.0:1775774942.651326:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774942.651329:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774942.651331:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1775774943.675283:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774943.675286:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774943.675289:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774946.619761:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774946.619768:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775774946.620103:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774946.620104:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775774946.620279:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774946.620281:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000020:00000001:2.0:1775774947.771368:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774947.771373:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774947.771377:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774951.739718:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774951.739722:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775774951.740071:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774951.740073:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775774951.740328:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774951.740330:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.0:1775774952.571264:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775774952.571270:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775774952.571274:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000020:00000001:2.0:1775774952.891291:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774952.891294:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774952.891297:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774956.859853:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774956.859856:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774956.860250:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774956.860253:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775774956.860562:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774956.860565:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000020:00000001:2.0:1775774958.011414:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774958.011419:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774958.011422:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774961.979707:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774961.979713:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774961.980089:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774961.980091:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775774961.980346:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774961.980347:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1775774963.131283:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774963.131286:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774963.131289:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774967.099811:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774967.099816:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774967.100250:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774967.100252:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775774967.100508:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774967.100510:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000020:00000001:2.0:1775774968.251306:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774968.251309:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774968.251312:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774972.219869:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774972.219873:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775774972.220243:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774972.220245:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775774972.220431:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774972.220433:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000020:00000001:2.0:1775774973.371342:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774973.371346:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774973.371349:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774977.339785:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774977.339788:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775774977.340071:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774977.340073:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775774977.340292:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774977.340294:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000020:00000001:2.0:1775774978.491334:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774978.491340:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774978.491344:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775774982.459329:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775774982.459337:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775774982.459341:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775774982.459348:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775774982.459358:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775774982.459364:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774982.459373:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775774982.459376:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775774982.459385:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775774982.459916:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774982.460175:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774982.460381:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:2.0:1775774983.611361:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774983.611368:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774983.611372:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775774987.579902:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774987.579905:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774987.580384:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774987.580387:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775774987.580724:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775774987.580727:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000020:00000001:2.0:1775774988.731407:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774988.731412:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774988.731415:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774992.699954:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774992.699959:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775774992.700538:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774992.700540:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775774992.700774:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774992.700777:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000020:00000001:2.0:1775774993.851366:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774993.851370:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774993.851372:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775774997.819906:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774997.819912:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775774997.820354:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774997.820356:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775774997.820582:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775774997.820584:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000020:00000001:2.0:1775774998.971408:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775774998.971412:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775774998.971416:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775002.939273:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775002.939281:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775002.939284:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775002.939288:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775002.939293:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775002.939302:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11885 00000400:00000200:2.0:1775775002.939314:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775002.939323:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775002.939327:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775002.939332:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775002.939336:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775002.939340:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775002.939343:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775002.939349:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775002.939351:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775002.939355:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775002.939360:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775002.939363:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775002.939367:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775002.939372:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775002.939374:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775002.939376:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775002.939379:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775002.939383:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775002.939387:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775002.939390:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775002.939393:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775002.939397:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775002.939400:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775002.939402:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775002.939404:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775002.939415:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775775002.939418:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775002.939423:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775002.939424:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775002.939428:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775002.939431:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775002.939436:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775002.939446:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775775002.939454:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775002.939465:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710306688/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775002.939476:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775002.939483:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775002.939487:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710306688/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775002.939527:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775002.939531:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775002.939534:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775002.939548:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775002.939550:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775002.939553:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775002.939556:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11885 00000400:00000200:2.0:1775775002.939560:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775002.939564:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775002.939566:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775002.939570:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775002.939573:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775002.939575:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775002.939577:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775002.939580:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775002.939581:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775002.939583:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775002.939588:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775002.939590:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775002.939595:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775002.939599:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775002.939602:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775002.939606:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775002.939608:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775002.939613:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775002.939619:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775002.939628:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775002.939631:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775002.939640:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775002.939646:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775002.939650:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775002.939658:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775002.939661:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775002.939664:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775002.939669:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775775002.939674:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775002.939680:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775002.939682:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775002.939683:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775002.939685:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775002.939689:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775002.939697:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775002.939705:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775002.939714:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710306816/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775002.939723:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775002.939726:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775002.939729:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710306816/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775002.939745:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775002.939749:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775002.939754:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775002.939757:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775002.939760:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775002.939768:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775002.939774:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11885 00000400:00000200:2.0:1775775002.939784:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775002.939791:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775002.939795:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775002.939800:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775002.939805:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775002.939810:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775002.939813:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775002.939815:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775002.939816:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775002.939818:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775002.939820:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775002.939824:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775002.939827:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775002.939830:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775002.939835:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775002.939841:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775002.939843:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775002.939845:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775002.939850:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775002.939854:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775002.939857:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775002.939861:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775002.939864:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775002.939868:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775002.939870:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775002.939873:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775002.939877:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775002.939882:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775775002.939888:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775002.939891:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775002.939893:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775002.939895:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775002.939898:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775002.939902:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775002.939908:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775002.939915:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775002.939927:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710306944/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775002.939948:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775002.939953:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775775002.939954:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775002.939961:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000100:00000040:2.0:1775775002.939962:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710306944/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775002.939986:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775002.939993:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775002.940000:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775002.940003:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775002.940008:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775002.940023:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11895) 00000800:00000200:0.1:1775775002.940476:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775002.940479:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775775002.940776:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775002.940778:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000020:00000001:2.0:1775775004.091269:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775004.091273:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775004.091277:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775008.059221:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775008.059229:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775008.059232:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775008.059235:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775008.059238:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775008.059244:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11890 00000400:00000200:2.0:1775775008.059252:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775008.059257:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775008.059259:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775008.059262:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775008.059265:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775008.059267:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775008.059268:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775008.059271:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775008.059273:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775008.059275:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775008.059277:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775008.059278:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775008.059280:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775008.059282:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775008.059283:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775008.059285:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775008.059286:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775008.059288:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775008.059290:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775008.059292:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775008.059294:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775008.059297:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775008.059299:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775008.059300:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775008.059302:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775008.059310:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775775008.059312:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775008.059315:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775008.059316:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775008.059319:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775008.059321:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775008.059324:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775008.059331:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775775008.059346:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775008.059354:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710307072/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775008.059367:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775008.059371:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775008.059374:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710307072/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775008.059392:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775008.059395:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775008.059397:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775008.059398:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775008.059399:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775008.059401:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775008.059403:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11890 00000400:00000200:2.0:1775775008.059405:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775008.059407:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775008.059409:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775008.059411:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775008.059412:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775008.059413:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775008.059415:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775008.059416:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775008.059417:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775008.059418:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775008.059420:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775008.059421:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775008.059422:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775008.059424:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775008.059425:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775008.059426:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775008.059427:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775008.059429:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775008.059432:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775008.059434:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775008.059436:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775008.059440:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775008.059442:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775008.059443:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775008.059459:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775008.059463:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775008.059466:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775008.059471:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775775008.059475:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775008.059479:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775008.059481:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775008.059486:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775008.059488:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775008.059490:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775008.059492:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775775008.059496:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775008.059505:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710307200/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775008.059524:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775008.059526:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775008.059532:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710307200/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775008.059544:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775008.059547:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775008.059552:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775008.059553:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775008.059555:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775008.059557:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775008.059561:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11890 00000400:00000200:2.0:1775775008.059564:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775008.059566:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775008.059571:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775008.059577:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775008.059580:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775008.059585:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775008.059589:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775008.059593:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775008.059597:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775008.059602:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775008.059606:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775008.059611:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775008.059617:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775008.059626:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775008.059632:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775008.059636:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775008.059638:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775008.059642:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775008.059646:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775008.059648:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775008.059649:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775008.059652:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775008.059656:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775008.059658:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775008.059660:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775008.059662:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775008.059664:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775008.059670:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775775008.059674:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775008.059678:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775008.059681:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775008.059684:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775008.059686:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775008.059688:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775008.059694:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775775008.059700:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775008.059707:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710307328/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775008.059722:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775008.059724:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775008.059729:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710307328/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775008.059747:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775008.059749:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775008.059752:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775008.059754:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775008.059758:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775008.059762:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11900) 00000800:00000200:0.1:1775775008.059773:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775008.059775:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775775008.060238:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775008.060240:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775775008.060537:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775008.060540:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000020:00000001:2.0:1775775009.211304:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775009.211308:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775009.211312:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775775013.179751:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775013.179756:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775775013.180219:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775013.180221:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775775013.180460:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775013.180462:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000020:00000001:2.0:1775775014.331423:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775014.331428:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775014.331431:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775775018.299776:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775018.299779:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775775018.300082:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775018.300084:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775775018.300314:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775018.300316:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000020:00000001:2.0:1775775019.451315:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775019.451320:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775019.451322:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775023.419258:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775023.419263:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775023.419265:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775023.419269:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775023.419272:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775023.419277:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11906 00000400:00000200:2.0:1775775023.419285:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775023.419290:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775023.419292:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775023.419294:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775023.419297:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775023.419299:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775023.419300:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775023.419303:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775023.419304:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775023.419306:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775023.419308:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775023.419310:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775023.419311:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775023.419313:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775023.419314:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775023.419315:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775023.419317:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775023.419319:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775023.419322:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775023.419323:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775023.419325:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775023.419328:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775023.419329:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775023.419331:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775023.419332:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775023.419339:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775775023.419342:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775023.419344:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775023.419346:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775023.419348:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775023.419350:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775023.419354:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775023.419360:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775775023.419365:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775023.419372:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710308224/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775023.419380:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775023.419383:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775023.419386:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710308224/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775023.419404:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775023.419407:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775023.419409:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775023.419410:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775023.419411:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775023.419413:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775023.419415:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11906 00000400:00000200:2.0:1775775023.419417:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775023.419419:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775023.419421:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775023.419423:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775023.419425:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775023.419427:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775023.419428:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775023.419430:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775023.419431:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775023.419433:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775023.419434:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775023.419436:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775023.419440:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775023.419444:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775023.419446:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775023.419449:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775023.419452:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775023.419454:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775023.419462:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775023.419467:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775023.419470:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775023.419475:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775023.419478:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775023.419482:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775023.419487:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775023.419493:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775023.419495:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775023.419499:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775775023.419502:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775023.419506:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775023.419509:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775023.419510:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775023.419511:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775023.419513:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775023.419520:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775775023.419524:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775023.419531:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710308352/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775023.419538:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775023.419542:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775023.419544:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710308352/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775023.419560:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775023.419564:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775023.419571:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775023.419574:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775023.419577:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775023.419580:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775023.419586:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11906 00000400:00000200:2.0:1775775023.419592:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775023.419597:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775023.419599:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775023.419605:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775023.419607:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775023.419608:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775023.419610:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775023.419615:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775023.419617:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775023.419618:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775023.419621:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775023.419626:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775023.419628:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775023.419634:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775023.419637:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775023.419642:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775023.419646:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775023.419649:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775023.419655:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775023.419659:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775023.419664:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775023.419669:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775023.419671:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775023.419673:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775023.419676:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775023.419681:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775023.419685:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775023.419691:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775775023.419694:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775023.419696:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775023.419697:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775023.419698:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775023.419701:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775023.419703:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775023.419706:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775023.419710:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775023.419717:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710308480/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1775775023.419722:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775023.419728:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1775775023.419734:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775023.419738:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775023.419744:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710308480/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775023.419757:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775023.419761:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775023.419764:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775023.419767:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775023.419770:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775023.419774:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11916) 00000800:00000200:1.1:1775775023.420032:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775023.420034:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775775023.420257:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775023.420259:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000020:00000001:2.0:1775775024.571311:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775024.571313:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775024.571316:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775028.539247:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775028.539260:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775028.539266:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775028.539271:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775028.539281:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775028.539295:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11911 00000400:00000200:2.0:1775775028.539309:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775028.539318:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775028.539322:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775028.539326:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775028.539331:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775028.539334:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775028.539337:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775028.539343:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775028.539345:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775028.539349:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775028.539352:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775028.539354:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775028.539356:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775028.539359:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775028.539360:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775028.539362:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775028.539364:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775028.539366:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775028.539369:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775028.539371:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775028.539374:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775028.539377:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775028.539379:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775028.539380:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775028.539382:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775028.539392:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775775028.539395:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775028.539400:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775028.539401:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775028.539405:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775028.539407:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775028.539411:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775028.539420:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775775028.539427:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775028.539436:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710308608/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775028.539446:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775028.539451:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775028.539454:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710308608/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775028.539487:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775028.539490:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775028.539493:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775028.539495:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775028.539496:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775028.539498:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775028.539500:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11911 00000400:00000200:2.0:1775775028.539503:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775028.539505:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775028.539507:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775028.539510:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775028.539512:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775028.539514:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775028.539515:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775028.539517:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775028.539518:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775028.539520:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775028.539522:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775028.539525:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775028.539528:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775028.539531:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775028.539535:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775028.539540:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775028.539546:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775028.539550:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775028.539556:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775028.539562:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775028.539571:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775028.539582:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775028.539592:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775028.539599:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775028.539605:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775028.539614:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775028.539622:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775028.539631:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775775028.539637:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775028.539643:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775028.539646:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775028.539648:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775028.539650:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775028.539652:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775028.539656:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775775028.539662:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775028.539672:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710308736/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775028.539687:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775028.539691:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775028.539698:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710308736/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775028.539727:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775028.539735:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775028.539740:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775028.539742:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775028.539745:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775028.539753:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775028.539762:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11911 00000400:00000200:2.0:1775775028.539778:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775028.539790:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775028.539793:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775028.539801:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775028.539808:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775028.539811:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775028.539813:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775028.539819:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775028.539820:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775028.539821:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775028.539823:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775028.539824:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775028.539825:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775028.539836:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775028.539838:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775028.539839:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775028.539847:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775028.539854:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775028.539858:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775028.539863:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775028.539869:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775028.539876:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775028.539883:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775028.539888:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775028.539893:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775028.539902:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775028.539907:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775028.539914:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775775028.539924:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775028.539930:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775028.539932:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775028.539934:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775028.539935:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775028.539937:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775028.539945:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775775028.539953:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775028.539966:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710308864/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775028.539984:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775028.539986:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775028.539993:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710308864/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775028.540007:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775028.540011:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775028.540023:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775028.540033:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775028.540044:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775028.540050:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11921) 00000800:00000200:1.1:1775775028.540051:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775028.540057:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775775028.540531:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775028.540534:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775775028.540825:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775028.540828:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1775775029.691351:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775029.691354:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775029.691357:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775775033.659797:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775033.659802:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775775033.660200:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775033.660202:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775775033.660391:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775033.660393:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000020:00000001:2.0:1775775034.811323:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775034.811326:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775034.811329:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775038.779240:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775038.779246:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775038.779249:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775038.779252:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775038.779255:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775038.779263:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11921 00000400:00000200:2.0:1775775038.779272:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775038.779278:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775038.779281:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775038.779285:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775038.779288:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775038.779291:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775038.779293:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775038.779297:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775038.779299:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775038.779301:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775038.779303:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775038.779305:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775038.779307:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775038.779309:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775038.779310:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775038.779311:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775038.779312:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775038.779314:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775038.779316:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775038.779318:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775038.779321:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775038.779323:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775038.779325:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775038.779327:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775038.779329:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775038.779336:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775775038.779339:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775038.779343:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775038.779345:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775038.779348:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775038.779351:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775038.779355:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775038.779364:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775038.779371:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775038.779380:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710309376/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775038.779390:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775038.779395:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775038.779400:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710309376/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775038.779422:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775038.779424:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775038.779427:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775038.779428:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775038.779429:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775038.779431:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775038.779433:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11921 00000400:00000200:2.0:1775775038.779436:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775038.779439:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775038.779441:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775038.779444:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775038.779446:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775038.779449:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775038.779451:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775038.779452:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775038.779454:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775038.779457:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775038.779459:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775038.779462:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775038.779465:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775038.779467:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775038.779468:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775038.779470:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775038.779472:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775038.779475:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775038.779481:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775038.779485:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775038.779489:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775038.779495:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775038.779498:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775038.779500:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775038.779505:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775038.779509:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775038.779513:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775038.779516:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775775038.779520:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775038.779525:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775038.779527:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775038.779530:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775038.779530:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775038.779532:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775038.779540:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775038.779545:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775038.779551:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710309504/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775038.779558:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775038.779559:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775038.779562:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710309504/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775038.779569:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775038.779572:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775038.779575:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775038.779576:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775038.779579:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775038.779583:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775038.779587:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11921 00000400:00000200:2.0:1775775038.779592:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775038.779597:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775038.779601:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775038.779606:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775038.779608:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775038.779613:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775038.779617:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775038.779622:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775038.779623:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775038.779626:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775038.779627:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775038.779630:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775038.779632:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775038.779636:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775038.779637:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775038.779639:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775038.779640:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775038.779644:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775038.779648:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775038.779650:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775038.779651:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775038.779653:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775038.779655:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775038.779658:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775038.779660:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775038.779662:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775038.779664:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775038.779669:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775775038.779672:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775038.779675:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775038.779676:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775038.779677:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775038.779679:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775038.779681:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775038.779685:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775775038.779688:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775038.779693:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710309632/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775038.779704:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775038.779707:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775038.779712:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710309632/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775038.779718:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775038.779721:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775038.779724:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775038.779726:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775038.779730:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775038.779736:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11931) 00000800:00000200:1.1:1775775038.779768:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775038.779771:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775775038.780131:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775038.780150:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775775038.780400:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775038.780402:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000020:00000001:2.0:1775775039.931336:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775039.931340:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775039.931342:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775775043.899789:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775043.899814:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775775043.900260:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775043.900263:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775775043.900463:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775043.900465:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000020:00000001:2.0:1775775045.051365:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775045.051372:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775045.051377:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775775049.019909:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775049.020311:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775049.020496:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:2.0:1775775050.171304:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775050.171311:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775050.171316:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775775052.091467:0:2400:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:2.0:1775775052.091522:0:2400:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1775775052.091534:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775052.091537:0:2400:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:2.0:1775775052.091542:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775052.091546:0:2400:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:2.0:1775775052.091551:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775052.091555:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230a00. 00000400:00000200:2.0:1775775052.091560:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775052.091565:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775052.091571:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f37f8 00000400:00000010:2.0:1775775052.091573:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f37f8. 00000100:00000001:2.0:1775775052.091580:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775052.091588:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f622e062680 x1862022710310144/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775052.091619:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775052.091625:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e062680 x1862022710310144/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775074 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775052.091633:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775052.091635:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775775052.091646:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775052.091648:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230700. 00000400:00000200:2.0:1775775052.091652:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775052.091655:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775052.091659:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:2.0:1775775052.091661:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:2.0:1775775052.091663:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775052.091668:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f622e060380 x1862022710310272/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775052.091677:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775052.091696:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e060380 x1862022710310272/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775074 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775052.091704:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775052.091706:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775775052.091713:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775052.091716:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:2.0:1775775052.091719:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775052.091722:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775052.091726:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:2.0:1775775052.091728:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:2.0:1775775052.091730:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775052.091735:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f622e063800 x1862022710310400/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775052.091742:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775052.091745:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e063800 x1862022710310400/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775074 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775052.091750:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775052.091752:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:2.0:1775775054.139280:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775054.139285:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775054.139288:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775054.139290:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775054.139293:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775054.139299:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11936 00000400:00000200:2.0:1775775054.139306:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775054.139311:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775054.139313:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775054.139315:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775054.139319:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775054.139320:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775054.139322:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775054.139324:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775054.139326:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775054.139328:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775054.139330:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775054.139332:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775054.139333:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775054.139335:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775054.139337:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775054.139338:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775054.139339:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775054.139341:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775054.139343:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775054.139345:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775054.139347:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775054.139349:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775054.139351:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775054.139353:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775054.139354:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775054.139359:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775775054.139361:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775054.139364:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775054.139365:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775054.139367:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775054.139369:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775054.139373:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775054.139378:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775775054.139383:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775054.139390:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710310528/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775054.139398:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775054.139403:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775054.139408:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710310528/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775054.139432:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775054.139436:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775054.139439:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775054.139441:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775054.139442:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775054.139445:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775054.139448:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11936 00000400:00000200:2.0:1775775054.139452:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775054.139455:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775054.139458:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775054.139461:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775054.139463:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775054.139465:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775054.139467:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775054.139469:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775054.139471:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775054.139472:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775054.139474:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775054.139476:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775054.139477:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775054.139480:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775054.139482:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775054.139485:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775054.139487:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775054.139489:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775054.139492:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775054.139495:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775054.139497:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775054.139502:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775054.139505:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775054.139506:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775054.139510:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775054.139513:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775054.139515:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775054.139520:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775775054.139522:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775054.139525:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775054.139527:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775054.139529:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775054.139531:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775054.139534:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775054.139540:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775775054.139544:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775054.139568:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710310656/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775054.139598:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775054.139603:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775054.139611:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710310656/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775054.139630:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775054.139639:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775054.139662:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775054.139665:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775054.139667:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775054.139670:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775054.139677:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11936 00000400:00000200:2.0:1775775054.139683:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775054.139690:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775054.139694:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775054.139699:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775054.139703:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775054.139705:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775054.139708:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775054.139713:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775054.139714:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775054.139716:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775054.139719:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775054.139723:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775054.139728:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775054.139735:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775054.139739:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775054.139744:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775054.139750:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775054.139755:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775054.139761:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775054.139768:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775054.139771:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775054.139777:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775054.139783:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775054.139786:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775054.139789:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775054.139795:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775054.139798:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775054.139802:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775775054.139806:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775054.139811:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775054.139814:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775054.139817:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775054.139819:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775054.139822:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775054.139827:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775775054.139833:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775054.139843:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710310784/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775054.139859:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775054.139864:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775054.139871:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710310784/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775054.139893:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775054.139899:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775054.139905:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775054.139909:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775054.139914:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775054.139920:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11946) 00000800:00000200:1.1:1775775054.139930:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775054.139933:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775775054.140364:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775054.140366:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775775054.140695:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775054.140698:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000020:00000001:2.0:1775775055.291333:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775055.291336:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775055.291339:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775059.259263:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775059.259274:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775059.259277:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775059.259282:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775775059.259285:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775059.259286:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775059.259295:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11941 00000400:00000200:2.0:1775775059.259306:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775059.259316:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775059.259320:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775059.259324:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775059.259338:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775059.259341:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775059.259343:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775059.259348:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775059.259350:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775059.259354:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775059.259357:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775059.259361:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775059.259364:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775059.259367:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775059.259370:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775059.259372:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775059.259374:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775059.259377:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775059.259380:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775059.259383:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775059.259386:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775059.259389:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775059.259392:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775059.259394:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775059.259396:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775059.259406:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775775059.259410:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775059.259416:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775059.259418:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775059.259421:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775059.259425:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775059.259431:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775059.259442:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775059.259451:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775059.259463:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710310912/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775059.259477:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775059.259483:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775059.259488:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710310912/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775059.259514:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775059.259517:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775059.259520:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775059.259521:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775059.259523:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775059.259525:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775059.259527:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11941 00000400:00000200:2.0:1775775059.259531:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775059.259535:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775059.259537:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775059.259541:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775059.259543:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775059.259546:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775059.259547:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775059.259550:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775059.259574:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775059.259577:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775059.259582:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775059.259584:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775059.259590:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775059.259595:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775059.259598:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775059.259599:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775059.259603:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775059.259607:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775059.259610:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775059.259618:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775059.259621:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775059.259627:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775059.259635:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775059.259642:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775059.259649:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775059.259656:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775059.259659:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775059.259668:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775775059.259671:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775059.259673:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775059.259675:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775059.259683:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775059.259685:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775059.259689:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775059.259693:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775059.259698:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775059.259708:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710311040/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775059.259717:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775059.259722:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775059.259727:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710311040/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775059.259739:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775059.259741:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775059.259745:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775059.259749:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775059.259752:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775059.259758:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775059.259761:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11941 00000400:00000200:2.0:1775775059.259770:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775059.259778:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775059.259781:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775059.259784:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775059.259792:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775059.259794:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775059.259796:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775059.259798:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775059.259799:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775059.259803:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775059.259807:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775059.259809:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775059.259812:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775059.259815:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775059.259821:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775059.259824:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775059.259830:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775059.259833:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775059.259843:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775059.259845:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775059.259847:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775059.259850:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775059.259853:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775059.259858:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775059.259860:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775059.259867:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775059.259874:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775059.259879:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775775059.259886:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775059.259890:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775059.259894:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775059.259900:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775059.259904:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775059.259907:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775059.259914:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775775059.259919:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775059.259931:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710311168/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775059.259956:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775059.259962:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775059.259972:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710311168/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775059.260017:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775059.260027:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775059.260032:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775059.260035:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775059.260042:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775059.260051:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11951) 00000800:00000200:1.1:1775775059.260096:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775059.260100:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775775059.260502:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775059.260505:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000100:00000001:3.0:1775775059.260630:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775059.260632:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775059.260635:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775059.260637:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775059.260641:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775059.260833:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775059.260835:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00010000:00000001:3.0:1775775059.387260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775059.387267:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775059.387273:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775059.387275:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775059.387279:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775059.387281:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775060.283260:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775060.283267:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775060.283269:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775060.283275:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775060.283279:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775060.283282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775060.283284:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775060.283286:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775060.283288:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775060.411217:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775060.411223:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775060.411225:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775060.411228:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775060.411232:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775060.411234:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775060.411235:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775060.411237:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775060.411240:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775060.411241:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775060.411244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775060.411245:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775060.411293:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775060.411296:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775060.411299:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775061.307235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775061.307240:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775061.307241:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775061.307247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775061.307250:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775061.307253:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775061.307256:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775061.307258:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775061.307260:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775061.435237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775061.435252:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775061.435258:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775061.435260:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775061.435264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775061.435265:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775062.331249:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775062.331257:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775062.331260:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775062.331265:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775062.331268:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775062.331272:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775062.331275:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775062.331278:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775062.331280:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775062.459249:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775062.459257:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775062.459262:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775062.459263:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775062.459266:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775062.459267:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775063.355249:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775063.355256:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775063.355258:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775063.355264:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775063.355268:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775063.355272:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775063.355275:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775063.355278:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775063.355281:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775063.483244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775063.483256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775063.483262:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775063.483264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775063.483267:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775063.483269:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775063.483292:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775063.483294:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775063.483297:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775064.379226:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1775775064.379229:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775775064.379234:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775064.379237:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775064.379238:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775064.379242:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1775775064.379244:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:2.0:1775775064.379247:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775064.379248:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775064.379252:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775775064.379253:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775064.379257:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775064.379261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775775064.379263:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11946 00000100:00000001:3.0:1775775064.379264:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775775064.379276:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775064.379285:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775064.379289:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775064.379293:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775064.379297:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775064.379300:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775064.379303:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775064.379308:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775064.379310:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775064.379312:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775064.379315:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775064.379317:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775064.379319:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775064.379322:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775064.379323:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775064.379324:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775064.379325:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775064.379328:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775064.379330:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775064.379332:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775064.379334:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775064.379337:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775064.379339:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775064.379341:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775064.379343:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775064.379354:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775775064.379357:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775064.379361:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775064.379362:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775064.379365:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775064.379368:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775064.379372:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775064.379380:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775064.379387:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775064.379396:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710311296/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775064.379406:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775064.379411:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775064.379415:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710311296/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775064.379433:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775064.379435:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775064.379438:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775064.379439:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775064.379440:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775064.379443:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775064.379444:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11946 00000400:00000200:2.0:1775775064.379448:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775064.379450:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775064.379452:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775064.379455:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775064.379458:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775064.379459:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775064.379461:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775064.379477:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775064.379479:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775064.379480:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775064.379482:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775064.379484:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775064.379486:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775064.379490:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775064.379493:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775064.379495:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775064.379497:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775064.379499:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775064.379501:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775064.379503:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775064.379504:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775064.379509:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775064.379512:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775064.379514:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775064.379521:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775064.379525:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775064.379528:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775064.379531:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775775064.379533:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775064.379538:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775064.379539:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775064.379544:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775064.379548:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775064.379553:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775064.379562:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775064.379571:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775064.379588:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710311424/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775064.379614:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775064.379619:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775064.379630:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710311424/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775064.379648:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775064.379652:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775064.379656:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775064.379659:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775064.379662:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775064.379666:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775064.379670:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11946 00000400:00000200:2.0:1775775064.379675:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775064.379681:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775064.379690:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775064.379694:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775064.379703:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775064.379706:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775064.379708:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775064.379717:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775064.379722:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775064.379729:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775064.379731:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775064.379737:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775064.379746:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775064.379759:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775064.379765:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775064.379775:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775064.379782:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775064.379793:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775064.379803:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775064.379808:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775064.379813:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775064.379817:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775064.379819:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775064.379827:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775064.379830:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775064.379834:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775064.379843:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775064.379849:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775775064.379854:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775064.379868:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775064.379873:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775064.379877:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775064.379879:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775064.379886:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775064.379896:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775775064.379906:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775064.379922:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710311552/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1775775064.379927:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775064.379932:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000100:00000001:2.0:1775775064.379965:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775064.379971:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775064.379983:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710311552/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775064.380017:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775064.380025:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775064.380039:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775064.380050:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775064.380059:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775064.380072:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11957) 00000800:00000200:1.1:1775775064.380347:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775064.380349:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1775775064.380586:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775064.380588:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00010000:00000001:3.0:1775775064.507288:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775064.507296:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775064.507305:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775064.507307:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775064.507311:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775064.507313:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775065.403281:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775065.403292:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775065.403294:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775065.403300:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775065.403304:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775065.403309:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775065.403312:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775065.403314:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775065.403316:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775065.531240:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775065.531248:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775065.531253:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775065.531257:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775065.531259:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775065.531262:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775065.531265:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775065.531267:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775065.531269:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775065.531270:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775065.531274:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775065.531275:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775065.531276:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775065.531278:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775065.531279:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775065.531280:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775065.531281:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775065.531282:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775065.531391:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775065.531396:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775065.531401:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775066.427219:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775066.427224:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775066.427226:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775066.427231:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775066.427233:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775066.427236:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775066.427239:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775066.427240:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775066.427242:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775066.555244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775066.555249:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775066.555253:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775066.555254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775066.555257:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775066.555258:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775067.451230:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775067.451236:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775067.451237:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775067.451242:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775067.451245:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775067.451247:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775067.451249:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775067.451251:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775067.451253:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775067.579240:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775067.579246:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775067.579252:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775067.579253:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775067.579257:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775067.579258:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775068.475276:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775068.475284:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775068.475286:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775068.475291:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775068.475296:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775068.475299:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775068.475301:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775068.475303:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775068.475305:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775068.603284:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775068.603292:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775068.603297:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775068.603298:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775068.603301:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775068.603302:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775068.603328:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775068.603331:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775068.603333:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775069.499233:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775069.499240:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775069.499242:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775069.499247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775069.499250:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775069.499253:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775069.499256:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775069.499258:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775069.499262:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775069.499712:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775069.499719:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775775069.500416:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775069.500418:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775775069.500614:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775069.500616:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00010000:00000001:3.0:1775775069.627224:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775069.627228:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775069.627233:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775069.627234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775069.627237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775069.627238:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775070.523235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775070.523243:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775070.523245:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775070.523251:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775070.523255:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775070.523260:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775070.523263:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775070.523265:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775070.523277:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775070.651246:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775070.651254:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775070.651257:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775070.651260:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775070.651265:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775070.651268:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775070.651270:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775070.651273:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775070.651276:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775070.651277:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775070.651280:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775070.651281:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775070.651338:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775070.651342:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775070.651344:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775071.547304:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775071.547314:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775071.547317:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775071.547324:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775071.547330:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775071.547334:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775071.547355:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775071.547359:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775071.547362:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775071.675276:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775071.675285:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775071.675292:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775071.675294:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775071.675299:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775071.675300:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775072.571286:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775072.571296:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775072.571299:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775072.571307:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775072.571312:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775072.571317:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775072.571334:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775072.571339:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775072.571343:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775072.699264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775072.699271:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775072.699278:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775072.699280:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775072.699284:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775072.699285:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775073.595237:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775073.595243:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775073.595244:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775073.595249:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775073.595251:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775073.595255:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775073.595257:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775073.595259:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775073.595261:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775073.723248:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775073.723253:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775073.723258:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775073.723260:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775073.723264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775073.723266:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775073.723292:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775073.723295:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775073.723298:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775074.619208:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775074.619214:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775074.619216:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775074.619221:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775074.619224:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775074.619227:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775074.619229:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775074.619231:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775074.619233:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775074.619240:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775074.619248:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775074.619252:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775074.619256:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775074.619260:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775074.619268:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11957 00000400:00000200:2.0:1775775074.619277:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775074.619295:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775074.619299:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775074.619303:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775074.619306:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775074.619309:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775074.619312:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775074.619316:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775074.619317:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775074.619320:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775074.619324:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775074.619326:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775074.619329:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775074.619332:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775074.619334:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775074.619336:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775074.619338:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775074.619341:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775074.619345:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775074.619348:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775074.619352:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775074.619355:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775074.619358:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775074.619360:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775074.619362:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775074.619371:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775775074.619374:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775074.619377:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775074.619379:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775074.619381:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775074.619383:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775074.619386:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775074.619392:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775775074.619397:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775074.619404:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710312064/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775074.619412:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775074.619416:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775074.619420:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710312064/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775074.619438:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775074.619440:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775074.619443:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775074.619444:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775074.619445:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775074.619447:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775074.619449:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11957 00000400:00000200:2.0:1775775074.619452:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775074.619454:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775074.619456:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775074.619458:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775074.619460:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775074.619462:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775074.619464:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775074.619465:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775074.619466:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775074.619467:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775074.619469:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775074.619471:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775074.619473:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775074.619477:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775074.619482:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775074.619486:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775074.619490:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775074.619495:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775074.619503:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775074.619508:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775074.619511:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775074.619517:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775074.619523:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775074.619526:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775074.619532:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775074.619536:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775074.619538:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775074.619542:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775775074.619546:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775074.619550:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775074.619552:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775074.619554:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775074.619556:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775074.619558:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775074.619562:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775775074.619567:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775074.619573:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710312192/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775074.619581:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775074.619586:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775074.619590:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710312192/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775074.619617:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775074.619622:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775074.619631:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775074.619634:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775074.619636:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775074.619641:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775074.619647:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11957 00000400:00000200:2.0:1775775074.619656:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775074.619661:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775074.619664:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775074.619670:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775074.619673:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775074.619676:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775074.619680:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775074.619683:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775074.619685:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775074.619688:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775074.619691:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775074.619704:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775074.619707:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775074.619711:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775074.619713:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775074.619716:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775074.619720:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775074.619722:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775074.619727:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775074.619731:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775074.619734:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775074.619737:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775074.619742:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775074.619746:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775074.619754:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775074.619758:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775074.619761:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775074.619768:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775775074.619772:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775074.619777:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775074.619779:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775074.619782:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775074.619785:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775074.619788:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775074.619794:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775775074.619800:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775074.619808:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710312320/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1775775074.619821:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:2.0:1775775074.619825:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.1:1775775074.619827:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000100:00000001:2.0:1775775074.619828:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775074.619837:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710312320/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775074.619856:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775074.619863:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775074.619869:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775074.619874:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775074.619879:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775074.619897:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11967) 00000800:00000200:1.1:1775775074.620320:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775074.620323:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775775074.620647:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775074.620650:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00010000:00000001:3.0:1775775074.747256:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775074.747263:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775074.747269:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775074.747271:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775074.747273:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775074.747274:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775075.643241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775075.643247:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775075.643248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775075.643252:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775075.643255:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775075.643259:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775075.643261:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775075.643263:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775075.643265:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775075.771206:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775075.771210:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775075.771214:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775075.771217:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775075.771219:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775075.771222:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775075.771223:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775075.771225:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775075.771227:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775075.771228:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775075.771231:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775075.771232:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775075.771232:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775075.771233:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775075.771234:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775075.771235:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775075.771236:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775075.771237:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775075.771300:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775075.771302:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775075.771305:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775076.667219:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775076.667225:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775076.667227:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775076.667231:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775076.667234:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775076.667237:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775076.667239:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775076.667241:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775076.667244:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775076.795220:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775076.795226:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775076.795231:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775076.795232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775076.795235:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775076.795236:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775077.691292:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775077.691300:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775077.691302:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775077.691307:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775077.691312:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775077.691315:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775077.691318:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775077.691320:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775077.691323:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775077.819265:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775077.819271:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775077.819277:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775077.819278:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775077.819280:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775077.819281:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775078.715231:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775078.715240:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775078.715242:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775078.715249:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775078.715254:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775078.715258:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775078.715261:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775078.715264:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775078.715267:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775078.843281:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775078.843288:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775078.843293:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775078.843294:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775078.843297:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775078.843298:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775078.843323:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775078.843326:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775078.843329:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775079.739257:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775775079.739257:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775079.739263:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1775775079.739267:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775079.739268:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775079.739272:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1775775079.739273:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:2.0:1775775079.739276:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775079.739278:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775079.739281:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775775079.739282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775079.739285:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775079.739288:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775775079.739289:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11962 00000100:00000001:3.0:1775775079.739292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775775079.739302:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775079.739312:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775079.739316:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775079.739320:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775079.739323:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775079.739326:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775079.739329:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775079.739334:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775079.739336:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775079.739340:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775079.739343:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775079.739346:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775079.739348:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775079.739352:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775079.739354:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775079.739356:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775079.739359:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775079.739362:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775079.739366:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775079.739369:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775079.739372:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775079.739375:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775079.739379:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775079.739382:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775079.739391:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775079.739404:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775775079.739409:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775079.739413:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775079.739415:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775079.739419:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775079.739422:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775079.739428:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775079.739437:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775079.739444:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775079.739455:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710312448/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775079.739468:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775079.739473:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775079.739479:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710312448/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775079.739504:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775079.739507:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775079.739510:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775079.739511:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775079.739512:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775079.739514:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775079.739516:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11962 00000400:00000200:2.0:1775775079.739520:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775079.739522:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775079.739524:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775079.739527:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775079.739528:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775079.739530:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775079.739532:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775079.739535:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775079.739536:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775079.739537:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775079.739539:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775079.739541:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775079.739543:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775079.739545:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775079.739546:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775079.739549:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775079.739551:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775079.739555:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775079.739560:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775079.739565:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775079.739568:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775079.739577:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775079.739583:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775079.739588:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775079.739595:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775079.739603:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775079.739606:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775079.739615:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775775079.739620:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775079.739624:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775079.739627:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775079.739631:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775079.739635:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775079.739637:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775079.739642:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775775079.739649:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775079.739657:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710312576/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775079.739672:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775079.739678:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775079.739684:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710312576/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775079.739695:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775079.739698:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775079.739702:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775079.739703:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775079.739706:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775079.739710:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775079.739713:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11962 00000400:00000200:2.0:1775775079.739720:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775079.739726:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775079.739729:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775079.739737:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775079.739742:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775079.739747:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775079.739752:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775079.739757:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775079.739760:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775079.739764:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775079.739767:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775079.739772:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775079.739776:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775079.739782:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775079.739784:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775079.739788:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775079.739789:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775079.739793:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775079.739795:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775079.739797:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775079.739799:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775079.739802:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775079.739805:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775079.739806:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775079.739809:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775079.739812:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775079.739815:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775079.739818:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775775079.739820:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775079.739822:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775079.739824:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775079.739826:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775079.739828:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775079.739831:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775079.739834:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775775079.739838:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775079.739845:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710312704/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775079.739853:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775079.739856:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775079.739863:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710312704/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775079.739876:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775079.739881:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775079.739888:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775079.739891:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775079.739897:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775079.739901:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11972) 00000800:00000200:1.1:1775775079.739942:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775079.739952:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775775079.741966:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775079.741970:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775775079.742256:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775079.742258:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00010000:00000001:3.0:1775775079.867275:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775079.867281:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775079.867287:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775079.867288:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775079.867291:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775079.867292:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775080.763240:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775080.763245:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775080.763247:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775080.763251:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775080.763254:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775080.763257:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775080.763259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775080.763261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775080.763263:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775080.891263:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775080.891269:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775080.891271:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775080.891274:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775080.891277:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775080.891279:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775080.891281:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775080.891283:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775080.891286:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775080.891287:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775080.891289:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775080.891290:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775080.891362:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775080.891366:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775080.891370:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775081.787259:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775081.787274:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775081.787276:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775081.787283:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775081.787287:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775081.787291:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775081.787293:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775081.787295:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775081.787297:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775081.915232:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775081.915238:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775081.915243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775081.915244:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775081.915248:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775081.915250:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775082.811254:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775082.811263:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775082.811265:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775082.811272:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775082.811276:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775082.811281:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775082.811284:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775082.811287:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775082.811290:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775082.939251:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775082.939256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775082.939262:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775082.939263:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775082.939267:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775082.939268:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775083.835228:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775083.835233:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775083.835235:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775083.835239:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775083.835242:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775083.835244:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775083.835246:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775083.835249:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775083.835252:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775083.963208:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775083.963212:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775083.963216:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775083.963217:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775083.963219:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775083.963220:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775083.963235:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775083.963237:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775083.963239:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775084.859225:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775084.859233:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775084.859235:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775084.859241:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775084.859246:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775084.859252:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775084.859257:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775084.859261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775084.859265:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775084.859707:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775084.859710:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775775084.859987:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775084.859988:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775775084.860286:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775084.860288:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00010000:00000001:3.0:1775775084.987198:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775084.987203:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775084.987206:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775084.987207:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775084.987210:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775084.987211:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775085.883183:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775085.883188:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775085.883189:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775085.883192:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775085.883195:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775085.883197:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775085.883199:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775085.883201:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775085.883202:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775086.011235:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775086.011240:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775086.011243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775086.011246:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775086.011247:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775086.011250:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775086.011252:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775086.011254:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775086.011255:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775086.011256:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775086.011260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775086.011261:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775086.011261:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775086.011262:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775086.011263:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775086.011264:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775086.011265:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775086.011266:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775086.011325:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775086.011329:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775086.011332:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775086.907257:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775086.907263:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775086.907265:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775086.907270:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775086.907275:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775086.907279:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775086.907282:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775086.907285:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775086.907288:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775087.035219:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775087.035225:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775087.035230:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775087.035231:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775087.035233:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775087.035234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775087.931200:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775087.931206:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775087.931207:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775087.931213:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775087.931216:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775087.931219:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775087.931222:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775087.931224:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775087.931227:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775088.059261:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775088.059267:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775088.059272:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775088.059273:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775088.059276:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775088.059277:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775088.955215:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775088.955225:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775088.955227:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775088.955234:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775088.955239:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775088.955244:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775088.955247:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775088.955249:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775088.955252:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775089.083222:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775089.083229:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775089.083233:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775089.083234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775089.083236:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775089.083237:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775089.083257:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775089.083259:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775089.083261:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775089.979246:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775775089.979247:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775089.979254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1775775089.979256:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775089.979257:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775089.979259:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1775775089.979264:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775089.979269:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775775089.979270:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775089.979272:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775089.979274:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775775089.979275:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775089.979278:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775089.979280:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775089.979282:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11972 00000400:00000200:2.0:1775775089.979292:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775089.979299:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775089.979303:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775089.979308:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775089.979312:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775089.979314:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775089.979315:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775089.979318:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775089.979320:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775089.979322:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775089.979324:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775089.979326:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775089.979327:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775089.979329:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775089.979330:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775089.979331:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775089.979332:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775089.979335:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775089.979337:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775089.979340:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775089.979343:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775089.979345:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775089.979348:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775089.979350:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775089.979352:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775089.979361:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775775089.979363:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775089.979366:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775089.979367:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775089.979370:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775089.979373:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775089.979381:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775089.979389:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775775089.979395:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775089.979402:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710313216/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775089.979410:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775089.979414:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775089.979417:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710313216/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775089.979433:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775089.979435:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775089.979438:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775089.979439:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775089.979440:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775089.979442:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775089.979443:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11972 00000400:00000200:2.0:1775775089.979445:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775089.979447:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775089.979449:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775089.979451:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775089.979453:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775089.979455:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775089.979457:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775089.979458:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775089.979459:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775089.979460:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775089.979462:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775089.979463:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775089.979464:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775089.979466:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775089.979467:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775089.979471:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775089.979472:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775089.979475:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775089.979477:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775089.979480:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775089.979481:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775089.979487:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775089.979491:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775089.979495:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775089.979501:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775089.979506:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775089.979512:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775089.979517:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775775089.979522:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775089.979527:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775089.979530:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775089.979533:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775089.979535:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775089.979537:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775089.979546:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775775089.979552:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775089.979560:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710313344/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775089.979568:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775089.979569:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775089.979573:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710313344/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775089.979584:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775089.979588:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775089.979595:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775089.979605:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775089.979606:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775089.979609:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775089.979613:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11972 00000400:00000200:2.0:1775775089.979618:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775089.979624:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775089.979626:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775089.979630:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775089.979632:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775089.979636:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775089.979638:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775089.979641:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775089.979642:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775089.979646:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775089.979649:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775089.979652:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775089.979655:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775089.979661:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775089.979664:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775089.979669:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775089.979672:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775089.979676:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775089.979682:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775089.979687:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775089.979691:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775089.979694:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775089.979695:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775089.979696:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775089.979698:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775089.979700:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775089.979703:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775089.979708:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775775089.979718:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775089.979723:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775089.979725:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775089.979728:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775089.979730:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775089.979731:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775089.979736:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775775089.979740:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775089.979747:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710313472/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775089.979759:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775089.979762:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775089.979766:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710313472/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1775775089.979766:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775089.979771:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1775775089.979774:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775089.979776:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775089.979783:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775089.979787:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775089.979793:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775089.979798:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11982) 00000800:00000200:1.1:1775775089.980127:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775089.980129:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775775089.980354:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775089.980355:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00010000:00000001:3.0:1775775090.107267:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775090.107274:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775090.107278:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775090.107279:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775090.107282:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775090.107283:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775091.003265:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775091.003272:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775091.003274:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775091.003279:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775091.003282:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775091.003287:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775091.003289:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775091.003291:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775091.003293:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775091.131288:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775091.131296:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775091.131298:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775091.131301:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775091.131306:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775091.131308:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775091.131310:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775091.131311:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775091.131314:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775091.131316:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775091.131319:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775091.131320:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775091.131388:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775091.131392:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775091.131395:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775092.027276:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775092.027283:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775092.027285:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775092.027289:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775092.027293:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775092.027297:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775092.027299:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775092.027301:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775092.027303:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775092.155273:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775092.155279:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775092.155285:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775092.155285:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775092.155288:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775092.155289:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775093.051257:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775093.051263:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775093.051265:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775093.051269:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775093.051272:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775093.051275:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775093.051277:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775093.051279:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775093.051281:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775093.179250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775093.179256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775093.179259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775093.179260:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775093.179264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775093.179264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775094.075226:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775094.075231:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775094.075232:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775094.075237:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775094.075239:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775094.075242:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775094.075244:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775094.075246:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775094.075248:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775094.203247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775094.203252:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775094.203257:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775094.203258:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775094.203261:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775094.203262:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775094.203282:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775094.203284:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775094.203286:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775095.099248:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775095.099255:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775095.099259:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775095.099266:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775095.099273:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775095.099278:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775095.099283:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775095.099287:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775095.099292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775095.099807:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775095.099812:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775775095.100124:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775095.100125:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775775095.100334:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775095.100336:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00010000:00000001:3.0:1775775095.227252:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775095.227258:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775095.227264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775095.227265:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775095.227270:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775095.227271:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775096.123260:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775096.123269:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775096.123272:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775096.123279:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775096.123283:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775096.123288:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775096.123291:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775096.123294:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775096.123297:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775096.251297:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775096.251306:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775096.251314:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775096.251318:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775096.251321:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775096.251325:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775096.251328:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775096.251331:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775096.251340:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775096.251343:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775096.251350:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775096.251352:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775096.251353:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775096.251354:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775096.251356:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775096.251357:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775096.251359:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775096.251360:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775096.251425:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775096.251429:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775096.251433:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775097.147241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775097.147248:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775097.147251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775097.147257:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775097.147261:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775097.147265:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775097.147268:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775097.147271:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775097.147274:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775097.275226:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775097.275232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775097.275237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775097.275238:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775097.275242:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775097.275243:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775098.171228:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775098.171236:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775098.171238:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775098.171244:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775098.171248:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775098.171251:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775098.171253:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775098.171256:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775098.171258:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775098.299266:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775098.299272:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775098.299277:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775098.299278:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775098.299281:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775098.299282:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775099.195260:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775099.195267:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775099.195269:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775099.195274:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775099.195278:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775099.195282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775099.195284:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775099.195286:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775099.195288:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775099.323245:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775099.323253:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775099.323260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775099.323261:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775099.323265:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775099.323267:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775099.323295:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775099.323299:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775099.323302:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775100.219232:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775100.219246:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775100.219249:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775100.219259:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775100.219266:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775100.219273:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775100.219283:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775100.219285:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775100.219289:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775100.219983:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775100.219986:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775775100.220352:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775100.220354:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775775100.220600:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775100.220602:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00010000:00000001:3.0:1775775100.347226:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775100.347233:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775100.347235:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775100.347238:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775100.347243:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775100.347245:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775100.347248:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775100.347250:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775100.347255:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775100.347256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775100.347260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775100.347261:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775101.243208:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775101.243213:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775101.243215:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775101.243219:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775101.243223:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775101.243227:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775101.243230:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775101.243232:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775101.243234:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775101.371230:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775101.371235:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775101.371241:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775101.371243:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775101.371247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775101.371249:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775101.371307:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775101.371310:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775101.371312:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775102.267259:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775102.267264:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775102.267266:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775102.267270:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775102.267273:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775102.267276:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775102.267279:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775102.267281:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775102.267283:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775102.395241:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775102.395249:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775102.395255:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775102.395256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775102.395260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775102.395262:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775103.291263:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775103.291272:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775103.291275:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775103.291282:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775103.291287:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775103.291292:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775103.291295:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775103.291297:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775103.291300:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775103.419256:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775103.419263:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775103.419269:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775103.419270:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775103.419274:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775103.419275:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775104.315227:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775104.315234:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775104.315236:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775104.315241:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775104.315246:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775104.315250:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775104.315252:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775104.315255:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775104.315258:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775104.443235:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775104.443240:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775104.443244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775104.443244:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775104.443247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775104.443248:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775104.443265:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775104.443267:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775104.443269:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775105.339232:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1775775105.339237:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775775105.339240:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1775775105.339244:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775105.339245:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775105.339248:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1775775105.339251:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:2.0:1775775105.339252:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775105.339257:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775775105.339258:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775105.339262:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775775105.339264:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 11987 00000100:00000001:3.0:1775775105.339267:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775105.339270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775775105.339274:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775775105.339274:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775775105.339282:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775105.339284:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775105.339287:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775105.339290:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775105.339292:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775105.339294:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775105.339297:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775105.339298:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775105.339300:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775105.339302:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775105.339304:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775105.339306:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775105.339309:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775105.339312:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775105.339313:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775105.339315:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775105.339317:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775105.339319:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775105.339321:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775105.339323:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775105.339326:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775105.339328:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775105.339329:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775105.339331:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775105.339338:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775775105.339340:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775105.339343:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775105.339345:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775105.339347:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775105.339350:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775105.339354:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775105.339363:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775775105.339369:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775105.339379:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710314368/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775105.339391:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775105.339396:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775105.339401:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710314368/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775105.339422:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775105.339426:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775105.339430:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775105.339431:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775105.339433:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775105.339436:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775105.339439:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11987 00000400:00000200:2.0:1775775105.339444:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775105.339447:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775105.339450:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775105.339454:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775105.339456:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775105.339459:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775105.339461:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775105.339464:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775105.339466:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775105.339468:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775105.339470:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775105.339472:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775105.339474:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775105.339478:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775105.339482:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775105.339487:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775105.339492:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775105.339495:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775105.339502:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775105.339507:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775105.339511:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775105.339517:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775105.339520:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775105.339523:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775105.339528:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775105.339532:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775105.339534:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775105.339538:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775775105.339541:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775105.339545:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775105.339547:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775105.339549:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775105.339551:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775105.339553:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775105.339558:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775775105.339561:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775105.339567:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710314496/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775105.339576:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775105.339578:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775105.339580:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710314496/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775105.339594:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775105.339596:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775105.339601:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775105.339604:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775105.339607:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775105.339609:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775105.339611:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 11987 00000400:00000200:2.0:1775775105.339618:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775105.339622:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775105.339627:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775105.339630:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775105.339633:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775105.339637:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775105.339642:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775105.339647:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775105.339648:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775105.339652:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775105.339655:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775105.339659:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775105.339665:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775105.339674:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775105.339680:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775105.339684:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775105.339687:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775105.339692:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775105.339694:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775105.339698:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775105.339700:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775105.339702:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775105.339706:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775105.339708:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775105.339712:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775105.339716:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775105.339717:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775105.339723:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775775105.339726:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775105.339728:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775105.339729:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775105.339732:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775105.339734:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775105.339736:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775105.339739:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775105.339745:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775105.339782:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710314624/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775105.339797:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775105.339802:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775105.339809:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710314624/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775105.339823:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775105.339826:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775105.339831:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775105.339833:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775105.339839:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775105.339846:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (11998) 00000800:00000200:1.1:1775775105.339854:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775105.339859:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775775105.340309:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775105.340311:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775775105.340650:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775105.340652:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00010000:00000001:3.0:1775775105.467242:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775105.467247:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775105.467250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775105.467253:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775105.467255:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775105.467257:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775105.467259:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775105.467261:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775105.467263:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775105.467264:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775105.467267:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775105.467267:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775105.467268:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775105.467270:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775105.467271:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775105.467272:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775105.467273:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775105.467274:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775106.363261:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775106.363266:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775106.363268:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775106.363272:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775106.363275:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775106.363280:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775106.363283:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775106.363286:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775106.363288:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775106.491234:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775106.491241:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775106.491248:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775106.491249:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775106.491253:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775106.491255:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775106.491310:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775106.491314:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775106.491318:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775107.387211:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775107.387217:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775107.387219:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775107.387223:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775107.387226:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775107.387229:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775107.387231:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775107.387233:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775107.387235:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775107.515228:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775107.515233:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775107.515237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775107.515238:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775107.515240:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775107.515241:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775108.411222:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775108.411228:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775108.411229:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775108.411234:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775108.411237:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775108.411239:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775108.411241:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775108.411243:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775108.411245:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775108.539207:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775108.539212:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775108.539216:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775108.539217:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775108.539220:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775108.539221:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775109.435245:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775109.435252:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775109.435254:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775109.435260:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775109.435264:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775109.435268:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775109.435271:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775109.435273:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775109.435276:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775109.563251:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775109.563256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775109.563261:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775109.563262:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775109.563264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775109.563265:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775109.563285:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775109.563287:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775109.563289:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775110.459242:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775110.459248:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775110.459250:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775110.459255:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775110.459258:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775110.459262:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775110.459266:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775110.459270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775110.459275:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775110.459768:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775110.459772:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775775110.460251:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775110.460254:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775775110.460560:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775110.460562:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00010000:00000001:3.0:1775775110.587222:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775110.587230:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775110.587233:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775110.587236:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775110.587241:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775110.587243:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775110.587245:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775110.587247:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775110.587250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775110.587251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775110.587254:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775110.587255:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775111.483234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775111.483240:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775111.483241:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775111.483245:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775111.483248:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775111.483251:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775111.483253:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775111.483255:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775111.483257:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775111.611297:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775111.611303:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775111.611309:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775111.611310:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775111.611313:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775111.611314:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775111.611380:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775111.611383:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775111.611387:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775112.507214:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775112.507222:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775112.507224:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775112.507230:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775112.507234:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775112.507238:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775112.507241:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775112.507244:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775112.507247:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775112.635214:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775112.635223:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775112.635228:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775112.635229:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775112.635232:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775112.635232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775113.531220:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775113.531227:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775113.531228:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775113.531233:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775113.531236:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775113.531239:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775113.531241:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775113.531243:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775113.531245:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775113.659244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775113.659250:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775113.659256:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775113.659257:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775113.659262:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775113.659263:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775114.555226:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775114.555231:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775114.555233:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775114.555237:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775114.555240:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775114.555243:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775114.555245:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775114.555246:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775114.555248:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775114.683243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775114.683250:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775114.683256:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775114.683258:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775114.683262:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775114.683264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775114.683287:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775114.683289:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775114.683292:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775115.579238:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775115.579246:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775115.579251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775115.579258:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775115.579270:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775115.579276:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775115.579281:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775115.579284:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775115.579292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775115.579782:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775115.580060:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775115.580224:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00010000:00000001:3.0:1775775115.707294:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775115.707300:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775115.707306:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775115.707309:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775115.707312:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775115.707315:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775115.707318:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775115.707320:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775115.707322:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775115.707324:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775115.707327:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775115.707328:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775115.707329:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775115.707330:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775115.707331:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775115.707332:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775115.707333:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775115.707334:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775116.603281:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775116.603286:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775116.603288:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775116.603292:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775116.603295:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775116.603298:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775116.603303:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775116.603305:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775116.603307:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775116.731222:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775116.731229:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775116.731236:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775116.731238:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775116.731241:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775116.731243:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775116.731311:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775116.731315:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775116.731319:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775117.627279:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775117.627284:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775117.627286:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775117.627289:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775117.627292:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775117.627295:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775117.627297:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775117.627299:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775117.627301:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775117.755264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775117.755269:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775117.755275:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775117.755276:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775117.755280:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775117.755282:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775118.651393:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775118.651399:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775118.651401:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775118.651408:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775118.651412:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775118.651416:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775118.651419:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775118.651422:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775118.651425:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1775775118.651507:0:2399:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:2.0:1775775118.651577:0:2399:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1775775118.651589:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775118.651594:0:2399:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:2.0:1775775118.651601:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775118.651605:0:2399:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:2.0:1775775118.651610:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775118.651614:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:2.0:1775775118.651618:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775118.651622:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775118.651630:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3ee0 00000400:00000010:2.0:1775775118.651633:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3ee0. 00000100:00000001:2.0:1775775118.651640:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775118.651650:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f622e061f80 x1862022710315136/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775118.651688:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775118.651694:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061f80 x1862022710315136/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775140 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775118.651702:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775118.651704:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775775118.651715:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775118.651718:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230700. 00000400:00000200:2.0:1775775118.651722:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775118.651725:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775118.651729:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3c38 00000400:00000010:2.0:1775775118.651731:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3c38. 00000100:00000001:2.0:1775775118.651735:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775118.651741:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f622e061500 x1862022710315264/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775118.651751:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775118.651758:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061500 x1862022710315264/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775140 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775118.651770:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775118.651773:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775775118.651784:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775118.651789:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230a00. 00000400:00000200:2.0:1775775118.651795:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775118.651803:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775118.651825:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:2.0:1775775118.651828:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:2.0:1775775118.651834:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775118.651842:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f622e063b80 x1862022710315392/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775118.651859:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775118.651868:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e063b80 x1862022710315392/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775140 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775118.651884:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775118.651902:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:3.0:1775775118.779268:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775118.779276:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775118.779289:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775118.779290:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775118.779293:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775118.779294:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775119.675320:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775119.675325:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775119.675326:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775119.675330:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775119.675333:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775119.675337:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775119.675339:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775119.675341:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775119.675343:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775119.803215:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775119.803226:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775119.803231:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775119.803232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775119.803234:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775119.803235:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775119.803253:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775119.803255:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775119.803257:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775120.699249:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775120.699256:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775120.699258:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775120.699261:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775120.699267:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775120.699275:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12003 00000400:00000200:2.0:1775775120.699296:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775775120.699297:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775120.699302:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:2.0:1775775120.699304:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:3.0:1775775120.699306:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775120.699310:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775775120.699312:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000040:2.0:1775775120.699317:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775120.699318:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775120.699323:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775775120.699324:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775775120.699327:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775120.699329:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775120.699331:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:3.0:1775775120.699333:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775775120.699335:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775120.699337:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000100:00000001:3.0:1775775120.699338:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775120.699340:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775120.699345:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775120.699348:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775120.699351:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775120.699355:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775120.699358:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775120.699360:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775120.699362:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775120.699365:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775120.699369:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775120.699372:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775120.699375:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775120.699379:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775120.699382:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775120.699384:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775120.699387:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775120.699394:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775775120.699397:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775120.699402:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775120.699404:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775120.699407:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775120.699410:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775120.699414:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775120.699421:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775775120.699429:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775120.699439:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710315520/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775120.699450:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775120.699455:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775120.699459:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710315520/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775120.699510:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775120.699514:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775120.699517:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775120.699519:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775120.699520:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775120.699523:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775120.699526:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12003 00000400:00000200:2.0:1775775120.699530:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775120.699533:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775120.699535:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775120.699538:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775120.699540:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775120.699542:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775120.699544:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775120.699547:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775120.699549:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775120.699551:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775120.699553:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775120.699555:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775120.699558:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775120.699561:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775120.699564:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775120.699566:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775120.699568:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775120.699570:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775120.699574:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775120.699576:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775120.699578:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775120.699584:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775120.699587:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775120.699589:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775120.699594:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775120.699598:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775120.699602:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775120.699610:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775775120.699615:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775120.699620:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775120.699622:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775120.699627:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775120.699629:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775120.699632:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775120.699640:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775775120.699647:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775120.699655:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710315648/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775120.699673:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775120.699679:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775120.699687:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710315648/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775120.699701:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775120.699706:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775120.699711:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775120.699714:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775120.699718:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775120.699722:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775120.699727:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12003 00000400:00000200:2.0:1775775120.699733:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775120.699738:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775120.699741:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775120.699746:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775120.699751:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775120.699756:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775120.699759:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775120.699763:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775120.699766:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775120.699770:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775120.699774:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775120.699779:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775120.699784:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775120.699792:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775120.699797:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775120.699800:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775120.699805:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775120.699809:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775120.699815:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775120.699818:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775120.699823:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775120.699829:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775120.699832:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775120.699836:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775120.699839:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775120.699843:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775120.699846:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775120.699852:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775775120.699857:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775120.699862:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775120.699865:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775120.699868:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775120.699872:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775120.699875:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775120.699882:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775775120.699888:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775120.699896:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710315776/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775120.699914:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775120.699918:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775120.699925:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710315776/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775120.699941:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775120.699946:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775775120.699950:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775775120.699953:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775775120.699956:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000100:00000040:2.0:1775775120.699958:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775120.699965:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775120.699973:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12013) 00000800:00000200:1.1:1775775120.700679:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775120.700681:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775775120.701258:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775120.701260:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00010000:00000001:3.0:1775775120.827216:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775120.827222:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775120.827224:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775120.827227:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775120.827231:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775120.827233:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775120.827235:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775120.827236:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775120.827239:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775120.827240:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775120.827243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775120.827244:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775121.723335:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775121.723340:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775121.723341:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775121.723346:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775121.723349:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775121.723352:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775121.723355:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775121.723358:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775121.723361:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775121.851253:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775121.851259:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775121.851264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775121.851266:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775121.851269:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775121.851270:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775121.851366:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775121.851370:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775121.851375:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775122.747231:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775122.747239:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775122.747241:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775122.747246:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775122.747250:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775122.747254:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775122.747257:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775122.747260:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775122.747262:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775122.875277:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775122.875285:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775122.875290:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775122.875291:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775122.875294:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775122.875295:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775123.771235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775123.771241:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775123.771242:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775123.771247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775123.771250:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775123.771253:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775123.771255:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775123.771258:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775123.771260:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775123.899250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775123.899256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775123.899264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775123.899265:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775123.899269:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775123.899270:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775124.795245:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775124.795253:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775124.795255:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775124.795261:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775124.795265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775124.795276:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775124.795279:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775124.795282:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775124.795285:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775124.923245:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775124.923251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775124.923255:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775124.923256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775124.923259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775124.923260:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775124.923281:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775124.923283:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775124.923285:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775125.819238:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775125.819246:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775125.819248:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775125.819258:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775125.819262:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775775125.819268:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1775775125.819269:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12008 00000100:00000001:3.0:1775775125.819278:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:2.0:1775775125.819282:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775775125.819282:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775775125.819290:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00100000:3.0:1775775125.819291:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:2.0:1775775125.819294:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1775775125.819298:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1775775125.819299:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775125.819306:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1775775125.819307:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775775125.819312:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775125.819316:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775125.819317:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:3.0:1775775125.819323:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1775775125.819324:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775125.819327:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775125.819334:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1775775125.819334:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775775125.819342:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775125.819346:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775125.819350:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775125.819354:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775125.819357:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775125.819360:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775125.819363:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775125.819366:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775125.819370:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775125.819374:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775125.819377:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775125.819380:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775125.819383:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775125.819387:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775125.819390:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775125.819411:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775775125.819421:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775125.819425:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775125.819428:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775125.819431:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775125.819434:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775125.819438:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775125.819446:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775775125.819452:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775125.819460:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710315904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775125.819469:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775125.819472:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775125.819476:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710315904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775125.819502:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775125.819505:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775125.819508:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775125.819509:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775125.819510:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775125.819512:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775125.819514:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12008 00000400:00000200:2.0:1775775125.819517:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775125.819519:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775125.819521:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775125.819523:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775125.819525:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775125.819526:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775125.819528:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775125.819531:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775125.819532:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775125.819533:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775125.819535:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775125.819537:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775125.819539:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775125.819540:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775125.819542:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775125.819544:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775125.819545:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775125.819547:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775125.819550:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775125.819555:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775125.819560:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775125.819566:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775125.819570:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775125.819575:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775125.819581:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775125.819586:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775125.819589:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775125.819595:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775775125.819599:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775125.819605:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775125.819608:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775125.819612:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775125.819614:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775125.819618:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775125.819626:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775775125.819633:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775125.819642:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710316032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775125.819651:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775125.819656:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775125.819663:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710316032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775125.819676:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775125.819681:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775125.819684:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775125.819696:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775125.819698:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775125.819702:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775125.819706:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12008 00000400:00000200:2.0:1775775125.819712:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775125.819717:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775125.819718:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775125.819722:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775125.819725:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775125.819729:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775125.819731:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775125.819732:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775125.819734:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775125.819736:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775125.819738:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775125.819742:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775125.819744:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775125.819748:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775125.819751:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775125.819757:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775125.819761:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775125.819764:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775125.819769:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775125.819772:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775125.819775:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775125.819779:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775125.819783:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775125.819787:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775125.819789:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775125.819792:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775125.819796:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775125.819799:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775775125.819801:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775125.819805:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775125.819807:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775125.819810:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775125.819812:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775125.819815:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775125.819819:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775125.819823:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775125.819829:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710316160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775125.819837:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775125.819840:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775125.819843:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710316160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775125.819854:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775125.819856:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775125.819862:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775125.819865:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775775125.819867:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775775125.819868:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775775125.819873:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000100:00000040:2.0:1775775125.819874:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12018) 00000800:00000200:1.1:1775775125.820290:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775125.820292:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775775125.820595:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775125.820597:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00010000:00000001:3.0:1775775125.947266:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775125.947273:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775125.947278:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775125.947281:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775125.947284:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775125.947287:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775125.947289:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775125.947292:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775125.947294:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775125.947296:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775125.947299:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775125.947300:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775125.947301:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775125.947302:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775125.947303:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775125.947304:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775125.947305:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775125.947306:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775126.843204:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775126.843211:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775126.843213:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775126.843219:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775126.843223:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775126.843227:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775126.843230:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775126.843233:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775126.843236:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775126.971221:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775126.971227:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775126.971232:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775126.971234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775126.971237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775126.971239:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775126.971308:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775126.971312:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775126.971315:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775127.867243:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775127.867249:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775127.867250:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775127.867254:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775127.867257:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775127.867260:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775127.867262:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775127.867264:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775127.867266:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775127.995230:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775127.995236:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775127.995240:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775127.995242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775127.995245:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775127.995246:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775128.891253:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775128.891261:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775128.891262:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775128.891267:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775128.891271:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775128.891275:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775128.891278:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775128.891280:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775128.891283:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775129.019288:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775129.019295:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775129.019302:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775129.019303:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775129.019306:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775129.019314:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775129.915245:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775129.915251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775129.915253:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775129.915257:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775129.915260:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775129.915263:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775129.915265:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775129.915267:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775129.915269:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775130.043270:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775130.043274:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775130.043278:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775130.043280:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775130.043282:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775130.043283:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775130.043310:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775130.043312:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775130.043316:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775130.939228:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775130.939234:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775130.939236:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775130.939241:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775130.939244:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775130.939247:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775130.939252:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775130.939254:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775130.939257:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775130.939865:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775130.939873:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775775130.940406:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775130.940408:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775775130.940658:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775130.940659:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00010000:00000001:3.0:1775775131.067245:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775131.067252:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775131.067254:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775131.067256:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775131.067260:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775131.067262:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775131.067263:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775131.067265:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775131.067268:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775131.067269:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775131.067272:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775131.067272:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775131.963349:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775131.963357:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775131.963360:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775131.963366:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775131.963370:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775131.963374:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775131.963377:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775131.963380:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775131.963383:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775132.091329:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775132.091338:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775132.091346:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775132.091348:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775132.091352:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775132.091354:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775132.091409:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775132.091412:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775132.091416:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775132.987287:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775132.987297:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775132.987300:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775132.987309:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775132.987315:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775132.987319:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775132.987322:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775132.987324:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775132.987327:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775133.115292:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775133.115299:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775133.115305:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775133.115306:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775133.115309:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775133.115310:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775134.011227:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775134.011234:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775134.011237:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775134.011242:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775134.011247:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775134.011264:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775134.011267:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775134.011270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775134.011273:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775134.139229:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775134.139235:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775134.139241:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775134.139243:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775134.139262:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775134.139263:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775135.035249:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775135.035257:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775135.035258:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775135.035265:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775135.035268:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775135.035272:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775135.035274:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775135.035277:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775135.035279:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775135.163223:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775135.163242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775135.163247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775135.163249:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775135.163251:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775135.163252:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775135.163273:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775135.163275:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775135.163277:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775136.059242:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775136.059249:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775136.059251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775136.059257:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775136.059262:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775136.059266:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775136.059270:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775136.059274:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775136.059277:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775136.059937:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775136.059946:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775775136.060468:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775136.060470:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775775136.060660:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775136.060661:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00010000:00000001:3.0:1775775136.187283:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775136.187289:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775136.187293:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775136.187296:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775136.187298:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775136.187300:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775136.187302:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775136.187304:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775136.187305:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775136.187307:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775136.187310:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775136.187310:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775136.187311:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775136.187312:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775136.187313:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775136.187314:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775136.187315:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775136.187316:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775137.083235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775137.083241:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775137.083243:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775137.083247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775137.083250:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775137.083254:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775137.083256:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775137.083258:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775137.083260:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775137.211237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775137.211242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775137.211246:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775137.211247:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775137.211250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775137.211251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775137.211346:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775137.211353:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775137.211358:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775138.107272:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775138.107278:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775138.107279:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775138.107284:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775138.107287:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775138.107290:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775138.107293:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775138.107294:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775138.107296:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775138.235254:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775138.235259:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775138.235263:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775138.235264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775138.235267:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775138.235268:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775139.131221:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775139.131229:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775139.131231:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775139.131235:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775139.131238:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775139.131242:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775139.131244:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775139.131246:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775139.131248:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775139.259219:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775139.259224:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775139.259228:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775139.259229:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775139.259233:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775139.259234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775140.155239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775140.155247:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775140.155249:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775140.155255:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775140.155258:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775140.155262:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775140.155264:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775140.155266:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775140.155269:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775140.283251:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775140.283258:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775140.283263:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775140.283264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775140.283267:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775140.283268:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775140.283291:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775140.283294:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775140.283297:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775141.179207:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775141.179214:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775141.179216:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775141.179222:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775141.179227:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775141.179230:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775141.179233:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775141.179235:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775141.179239:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775141.179660:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775141.179665:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775775141.180223:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775141.180226:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775775141.180581:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775141.180584:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00010000:00000001:3.0:1775775141.307227:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775141.307235:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775141.307238:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775141.307241:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775141.307247:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775141.307249:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775141.307252:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775141.307254:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775141.307259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775141.307261:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775141.307266:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775141.307267:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775142.203190:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775142.203196:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775142.203198:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775142.203202:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775142.203205:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775142.203208:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775142.203210:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775142.203212:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775142.203214:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775142.331221:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775142.331227:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775142.331233:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775142.331235:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775142.331239:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775142.331240:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775142.331338:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775142.331343:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775142.331348:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775143.227271:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775143.227281:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775143.227283:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775143.227290:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775143.227295:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775143.227301:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775143.227304:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775143.227308:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775143.227311:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775143.355226:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775143.355232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775143.355237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775143.355238:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775143.355241:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775143.355242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775144.251190:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775144.251196:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775144.251198:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775144.251202:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775144.251205:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775144.251208:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775144.251211:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775144.251212:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775144.251214:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775144.379240:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775144.379247:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775144.379252:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775144.379254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775144.379258:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775144.379260:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775145.275244:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775145.275253:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775145.275255:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775145.275262:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775145.275266:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775145.275270:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775145.275273:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775145.275275:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775145.275278:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775145.403207:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775145.403212:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775145.403216:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775145.403220:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775145.403222:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775145.403226:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775145.403229:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775145.403231:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775145.403233:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775145.403235:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775145.403239:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775145.403241:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775145.403243:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775145.403244:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775145.403246:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775145.403258:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775145.403260:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775145.403262:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775145.403285:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775145.403325:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775145.403327:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775146.299224:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775146.299233:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775146.299236:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775146.299245:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775146.299251:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775146.299255:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775146.299260:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775146.299263:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775146.299268:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775146.299287:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775146.299295:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775146.299298:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775146.299302:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775146.299305:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775146.299313:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12028 00000400:00000200:2.0:1775775146.299321:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775146.299328:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775146.299331:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775146.299334:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775146.299337:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775146.299339:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775146.299341:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775146.299344:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775146.299346:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775146.299348:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775146.299351:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775146.299352:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775146.299354:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775146.299356:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775146.299359:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775146.299360:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775146.299362:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775146.299365:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775146.299367:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775146.299369:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775146.299372:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775146.299374:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775146.299377:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775146.299380:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775146.299382:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775146.299394:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775775146.299399:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775146.299403:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775146.299405:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775146.299409:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775146.299413:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775146.299419:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775146.299431:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775775146.299440:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775146.299452:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710317440/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775146.299465:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775146.299471:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775146.299477:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710317440/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775146.299504:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775146.299507:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775146.299511:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775146.299513:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775146.299515:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775146.299518:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775146.299521:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12028 00000400:00000200:2.0:1775775146.299526:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775146.299530:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775146.299533:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775146.299537:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775146.299540:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775146.299543:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775146.299545:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775146.299548:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775146.299550:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775146.299553:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775146.299557:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775146.299562:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775146.299567:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775146.299574:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775146.299577:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775146.299580:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775146.299586:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775146.299594:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775146.299608:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775146.299621:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775146.299626:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775146.299645:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775146.299657:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775146.299663:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775146.299671:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775146.299679:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775146.299681:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775146.299687:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775775146.299695:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775146.299701:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775146.299703:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775146.299710:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775146.299714:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775146.299721:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775146.299729:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775146.299735:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775146.299748:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710317568/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775146.299778:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775146.299784:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775146.299791:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710317568/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775146.299813:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775146.299817:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775146.299828:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775146.299831:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775146.299833:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775146.299837:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775146.299845:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12028 00000400:00000200:2.0:1775775146.299856:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775146.299867:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775146.299876:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775146.299886:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775146.299892:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775146.299901:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775146.299908:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775146.299916:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775146.299923:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775146.299929:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775146.299933:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775146.299940:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775146.299947:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775146.299951:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775146.299957:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775146.299965:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775146.299973:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775146.299979:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775146.299986:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775146.299998:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775146.300007:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:1.1:1775775146.300011:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:2.0:1775775146.300013:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.1:1775775146.300015:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 02000000:00000001:2.0:1775775146.300021:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775146.300031:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775146.300064:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775146.300074:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775146.300078:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775146.300086:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775775146.300098:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775146.300108:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775146.300114:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775146.300122:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775146.300129:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775146.300196:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775146.300205:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775146.300213:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775146.300225:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710317696/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775146.300238:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775146.300244:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775146.300257:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710317696/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775146.300317:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775146.300327:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775146.300334:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775146.300339:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775146.300345:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775146.300355:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12039) 00000800:00000200:1.1:1775775146.300560:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775146.300562:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775775146.300768:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775146.300771:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00010000:00000001:3.0:1775775146.427245:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775146.427253:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775146.427261:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775146.427262:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775146.427266:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775146.427267:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775147.323247:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775147.323254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775147.323255:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775147.323261:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775147.323265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775147.323276:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775147.323279:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775147.323281:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775147.323284:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775147.451224:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775147.451230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775147.451236:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775147.451237:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775147.451241:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775147.451242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775147.451326:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775147.451330:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775147.451334:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775148.347208:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775148.347216:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775148.347217:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775148.347223:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775148.347227:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775148.347231:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775148.347234:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775148.347236:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775148.347238:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775148.475267:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775148.475275:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775148.475284:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775148.475295:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775148.475301:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775148.475302:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775149.371249:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775149.371255:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775149.371257:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775149.371261:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775149.371264:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775149.371268:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775149.371270:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775149.371272:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775149.371274:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775149.499230:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775149.499235:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775149.499239:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775149.499240:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775149.499243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775149.499244:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775150.395256:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775150.395263:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775150.395265:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775150.395271:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775150.395275:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775150.395279:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775150.395283:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775150.395285:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775150.395288:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775150.523249:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775150.523256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775150.523262:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775150.523264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775150.523268:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775150.523270:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775150.523296:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775150.523299:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775150.523302:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775151.419227:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775151.419233:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775151.419234:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775151.419238:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775151.419241:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775151.419244:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775151.419247:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775151.419249:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775151.419252:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775151.419753:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775151.419758:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775775151.420106:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775151.420108:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775775151.420316:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775151.420318:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00010000:00000001:3.0:1775775151.547237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775151.547243:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775151.547245:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775151.547248:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775151.547252:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775151.547253:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775151.547255:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775151.547257:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775151.547259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775151.547260:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775151.547263:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775151.547264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775152.443213:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775152.443219:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775152.443220:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775152.443225:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775152.443228:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775152.443231:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775152.443233:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775152.443235:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775152.443237:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775152.571252:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775152.571258:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775152.571263:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775152.571265:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775152.571269:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775152.571270:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775152.571336:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775152.571340:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775152.571344:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775153.467213:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775153.467220:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775153.467222:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775153.467228:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775153.467231:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775153.467235:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775153.467238:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775153.467240:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775153.467243:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775153.595214:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775153.595219:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775153.595225:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775153.595227:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775153.595231:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775153.595232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775154.491202:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775154.491207:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775154.491208:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775154.491213:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775154.491216:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775154.491219:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775154.491221:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775154.491223:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775154.491225:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775154.619219:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775154.619224:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775154.619228:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775154.619229:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775154.619232:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775154.619233:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775155.515214:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775155.515220:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775155.515222:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775155.515226:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775155.515229:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775155.515232:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775155.515235:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775155.515237:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775155.515239:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775155.643279:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775155.643287:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775155.643295:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775155.643300:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775155.643303:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775155.643308:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775155.643311:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775155.643314:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775155.643317:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775155.643319:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775155.643324:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775155.643326:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775155.643328:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775155.643329:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775155.643331:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775155.643333:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775155.643335:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775155.643337:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775155.643378:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775155.643382:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775155.643385:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775156.539232:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775156.539240:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775156.539243:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775156.539255:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775156.539263:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775156.539271:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775156.539278:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775156.539283:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775156.539291:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775156.539783:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775156.539789:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775775156.540189:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775156.540191:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775775156.540400:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775156.540402:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00010000:00000001:3.0:1775775156.667301:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775156.667310:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775156.667319:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775156.667330:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775156.667335:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775156.667337:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775157.563245:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775157.563251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775157.563252:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775157.563257:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775157.563260:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775157.563263:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775157.563265:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775157.563267:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775157.563269:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775157.691223:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775157.691230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775157.691237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775157.691239:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775157.691243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775157.691245:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775157.691310:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775157.691315:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775157.691318:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775158.587187:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775158.587194:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775158.587195:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775158.587199:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775158.587202:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775158.587205:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775158.587207:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775158.587208:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775158.587210:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775158.715235:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775158.715241:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775158.715247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775158.715249:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775158.715253:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775158.715254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775159.611209:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775159.611215:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775159.611216:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775159.611221:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775159.611224:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775159.611226:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775159.611228:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775159.611231:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775159.611233:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775159.739214:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775159.739219:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775159.739223:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775159.739224:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775159.739227:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775159.739228:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775160.635234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775160.635239:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775160.635240:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775160.635244:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775160.635247:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775160.635250:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775160.635252:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775160.635254:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775160.635256:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775160.763231:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775160.763238:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775160.763244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775160.763245:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775160.763249:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775160.763251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775160.763275:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775160.763279:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775160.763282:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775161.659196:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775161.659202:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775161.659203:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775161.659208:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775161.659213:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775161.659216:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775161.659220:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775161.659222:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775161.659228:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775161.659731:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775161.659734:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775775161.660028:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775161.660031:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775775161.660286:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775161.660288:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00010000:00000001:3.0:1775775161.787230:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775161.787237:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775161.787258:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775161.787262:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775161.787266:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775161.787268:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775161.787270:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775161.787273:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775161.787277:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775161.787279:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775161.787284:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775161.787286:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775162.683261:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775162.683269:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775162.683272:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775162.683277:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775162.683281:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775162.683285:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775162.683288:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775162.683291:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775162.683294:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775162.811225:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775162.811229:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775162.811233:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775162.811234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775162.811237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775162.811238:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775162.811292:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775162.811295:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775162.811298:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775163.707217:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775163.707223:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775163.707224:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775163.707229:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775163.707232:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775163.707234:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775163.707236:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775163.707238:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775163.707240:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775163.835255:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775163.835261:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775163.835267:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775163.835269:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775163.835281:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775163.835283:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775164.731244:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775164.731248:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775164.731250:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775164.731255:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775164.731257:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775164.731260:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775164.731263:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775164.731264:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775164.731266:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775164.859217:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775164.859223:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775164.859226:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775164.859227:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775164.859229:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775164.859230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775165.755206:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775165.755211:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775165.755213:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775165.755217:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775165.755220:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775165.755223:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775165.755225:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775165.755228:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775165.755230:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775165.883295:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775165.883302:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775165.883308:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775165.883312:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775165.883314:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775165.883317:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775165.883320:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775165.883323:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775165.883324:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775165.883326:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775165.883329:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775165.883330:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775165.883331:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775165.883333:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775165.883334:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775165.883335:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775165.883336:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775165.883337:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775165.883359:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775165.883361:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775165.883363:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775166.779241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775166.779247:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775166.779248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775166.779253:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775166.779257:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775775166.779265:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775775166.779267:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775166.779272:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1775775166.779275:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775166.779275:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775166.779279:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775166.779280:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775166.779284:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775166.779289:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775166.779296:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12049 00000400:00000200:2.0:1775775166.779306:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775166.779313:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775166.779316:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775166.779320:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775166.779323:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775166.779326:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775166.779328:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775166.779333:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775166.779335:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775166.779338:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775166.779341:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775166.779344:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775166.779346:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775166.779349:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775166.779351:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775166.779353:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775166.779355:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775166.779358:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775166.779360:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775166.779362:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775166.779365:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775166.779367:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775166.779370:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775166.779371:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775166.779373:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775166.779380:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775775166.779382:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775166.779385:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775166.779387:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775166.779389:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775166.779391:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775166.779394:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775166.779401:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775775166.779405:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775166.779412:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710318976/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775166.779420:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775166.779423:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775166.779427:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710318976/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775166.779447:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775166.779450:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775166.779453:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775166.779455:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775166.779456:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775166.779459:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775166.779462:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12049 00000400:00000200:2.0:1775775166.779466:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775166.779470:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775166.779472:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775166.779475:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775166.779478:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775166.779481:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775166.779483:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775166.779486:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775166.779488:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775166.779490:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775166.779492:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775166.779495:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775166.779499:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775166.779505:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775166.779508:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775166.779512:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775166.779516:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775166.779521:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775166.779523:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775166.779525:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775166.779527:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775166.779531:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775166.779535:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775166.779536:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775166.779540:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775166.779543:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775166.779552:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775166.779557:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775775166.779562:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775166.779566:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775166.779568:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775166.779571:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775166.779574:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775166.779578:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775166.779584:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775775166.779589:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775166.779597:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710319104/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775166.779605:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775166.779610:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775166.779618:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710319104/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775166.779634:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775166.779638:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775166.779646:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775166.779648:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775166.779650:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775166.779653:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775166.779660:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12049 00000400:00000200:2.0:1775775166.779671:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775166.779711:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775166.779715:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775166.779724:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775166.779728:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775166.779731:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775166.779736:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775166.779743:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775166.779747:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775166.779752:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775166.779756:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775166.779763:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775166.779766:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775166.779774:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775166.779780:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775166.779787:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775166.779792:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775166.779798:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775166.779804:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775166.779810:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775166.779814:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775166.779818:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775166.779823:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775166.779824:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:2.0:1775775166.779827:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:1.1:1775775166.779830:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000008:00100000:2.0:1775775166.779831:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775166.779837:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775166.779841:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775166.779848:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775775166.779852:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775166.779855:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775166.779857:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775166.779861:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775166.779863:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775166.779865:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775166.779869:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775775166.779876:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775166.779886:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710319232/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775166.779900:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775166.779904:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775166.779915:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710319232/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775166.779950:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775166.779954:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775166.779959:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775166.779964:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775166.779968:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775166.779974:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12059) 00000800:00000200:1.1:1775775166.780250:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775166.780252:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775775166.780496:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775166.780498:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00010000:00000001:3.0:1775775166.907221:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775166.907225:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775166.907229:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775166.907230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775166.907233:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775166.907234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775167.803229:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775167.803237:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775167.803239:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775167.803244:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775167.803248:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775167.803252:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775167.803256:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775167.803259:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775167.803262:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775167.931270:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775167.931276:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775167.931293:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775167.931294:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775167.931303:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775167.931304:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775167.931383:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775167.931388:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775167.931391:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775168.827253:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775168.827260:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775168.827262:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775168.827268:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775168.827272:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775168.827283:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775168.827286:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775168.827289:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775168.827292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775168.955209:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775168.955213:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775168.955217:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775168.955218:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775168.955220:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775168.955221:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775169.851262:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775169.851270:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775169.851272:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775169.851278:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775169.851283:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775169.851287:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775169.851291:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775169.851294:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775169.851297:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775169.979244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775169.979250:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775169.979256:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775169.979257:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775169.979261:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775169.979262:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775170.875283:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775170.875290:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775170.875292:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775170.875298:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775170.875302:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775170.875306:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775170.875309:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775170.875312:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775170.875315:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775171.003276:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775171.003285:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775171.003292:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775171.003294:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775171.003299:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775171.003301:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775171.003335:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775171.003339:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775171.003342:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775171.899247:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775171.899259:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775171.899260:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775171.899263:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775775171.899268:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1775775171.899270:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775171.899271:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775171.899280:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00100000:3.0:1775775171.899281:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775171.899285:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775775171.899289:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12054 00000100:00000001:3.0:1775775171.899289:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775171.899299:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775171.899301:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775171.899305:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775775171.899308:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775171.899316:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775171.899319:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775171.899322:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775171.899326:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775171.899328:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775171.899331:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775171.899334:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775171.899337:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775171.899339:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775171.899342:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775171.899344:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775171.899345:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775171.899348:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775171.899349:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775171.899350:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775171.899351:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775171.899354:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775171.899356:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775171.899358:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775171.899361:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775171.899363:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775171.899365:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775171.899367:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775171.899369:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775171.899380:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775775171.899383:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775171.899387:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775171.899388:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775171.899390:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775171.899393:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775171.899398:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775171.899407:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775775171.899413:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775171.899423:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710319360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775171.899432:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775171.899437:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775171.899441:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710319360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775171.899463:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775171.899465:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775171.899468:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775171.899470:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775171.899471:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775171.899473:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775171.899475:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12054 00000400:00000200:2.0:1775775171.899478:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775171.899481:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775171.899483:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775171.899485:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775171.899486:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775171.899488:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775171.899491:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775171.899493:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775171.899496:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775171.899498:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775171.899501:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775171.899503:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775171.899505:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775171.899509:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775171.899513:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775171.899516:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775171.899520:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775171.899523:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775171.899528:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775171.899531:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775171.899535:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775171.899541:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775171.899544:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775171.899546:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775171.899552:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775171.899556:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775171.899559:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775171.899561:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775775171.899565:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775171.899568:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775171.899570:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775171.899571:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775171.899572:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775171.899574:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775171.899579:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775775171.899582:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775171.899610:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710319488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775171.899620:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775171.899623:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775171.899629:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710319488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775171.899635:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775171.899638:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775171.899645:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775171.899647:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775171.899649:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775171.899652:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775171.899657:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12054 00000400:00000200:2.0:1775775171.899665:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775171.899669:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775171.899673:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775171.899679:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775171.899681:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775171.899685:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775171.899687:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775171.899689:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775171.899692:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775171.899695:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775171.899697:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775171.899698:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775171.899700:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775171.899703:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775171.899704:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775171.899707:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775171.899709:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775171.899713:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775171.899715:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775171.899716:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775171.899717:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775171.899720:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775171.899722:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775171.899726:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775171.899729:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775171.899733:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775171.899736:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775171.899741:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775775171.899744:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775171.899750:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775171.899753:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775171.899757:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775171.899760:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775171.899763:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775171.899771:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775171.899786:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775171.899794:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710319616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775171.899804:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775171.899806:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775171.899812:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710319616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775171.899823:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775171.899826:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775775171.899829:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775775171.899831:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775775171.899834:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000100:00000040:2.0:1775775171.899835:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775171.899846:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775171.899853:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12064) 00000800:00000200:1.1:1775775171.900171:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775171.900172:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775775171.900368:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775171.900371:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00010000:00000001:3.0:1775775172.027251:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775172.027259:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775172.027262:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775172.027265:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775172.027270:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775172.027272:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775172.027275:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775172.027277:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775172.027282:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775172.027283:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775172.027288:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775172.027289:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775172.923239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775172.923245:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775172.923246:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775172.923250:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775172.923253:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775172.923256:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775172.923258:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775172.923261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775172.923263:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775173.051259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775173.051264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775173.051270:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775173.051272:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775173.051275:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775173.051277:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775173.051358:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775173.051363:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775173.051367:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775173.947246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775173.947254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775173.947255:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775173.947261:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775173.947265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775173.947270:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775173.947272:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775173.947275:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775173.947277:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775174.075303:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775174.075311:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775174.075318:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775174.075320:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775174.075324:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775174.075325:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775174.971295:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775174.971304:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775174.971306:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775174.971314:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775174.971319:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775174.971324:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775174.971327:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775174.971331:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775174.971334:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775175.099244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775175.099251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775175.099256:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775175.099257:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775175.099261:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775175.099262:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775175.995292:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775175.995301:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775175.995303:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775175.995318:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775175.995324:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775175.995330:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775175.995334:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775175.995337:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775175.995340:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775176.123255:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775176.123260:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775176.123264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775176.123268:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775176.123269:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775176.123272:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775176.123274:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775176.123276:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775176.123278:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775176.123279:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775176.123282:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775176.123283:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775176.123284:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775176.123285:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775176.123286:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775176.123286:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775176.123288:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775176.123289:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775176.123313:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775176.123316:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775176.123318:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775177.019232:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775177.019239:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775177.019240:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775177.019247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775177.019251:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775177.019264:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775177.019267:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775177.019270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775177.019274:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775177.019890:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775177.019897:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775775177.020280:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775177.020281:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775775177.020479:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775177.020481:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00010000:00000001:3.0:1775775177.147240:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775177.147248:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775177.147254:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775177.147255:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775177.147260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775177.147261:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775178.043212:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775178.043220:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775178.043222:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775178.043228:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775178.043238:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775178.043242:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775178.043245:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775178.043247:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775178.043249:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775178.171253:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775178.171260:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775178.171265:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775178.171266:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775178.171269:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775178.171270:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775178.171356:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775178.171362:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775178.171366:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775179.067246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775179.067254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775179.067255:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775179.067262:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775179.067265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775179.067270:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775179.067272:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775179.067275:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775179.067277:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775179.195226:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775179.195232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775179.195238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775179.195247:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775179.195250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775179.195251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775180.091215:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775180.091222:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775180.091225:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775180.091231:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775180.091236:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775180.091240:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775180.091243:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775180.091247:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775180.091250:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775180.219234:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775180.219240:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775180.219245:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775180.219247:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775180.219251:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775180.219252:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775181.115259:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775181.115266:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775181.115268:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775181.115273:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775181.115276:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775181.115279:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775181.115281:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775181.115283:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775181.115285:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775181.243228:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775181.243234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775181.243239:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775181.243241:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775181.243244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775181.243246:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775181.243266:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775181.243268:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775181.243271:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775182.139243:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775182.139249:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775182.139252:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775182.139259:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775182.139265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775182.139271:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775182.139278:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775182.139283:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775182.139288:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775182.139876:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775182.140129:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775182.140308:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00010000:00000001:3.0:1775775182.267220:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775182.267228:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775182.267231:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775182.267234:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775182.267239:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775182.267241:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775182.267243:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775182.267246:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775182.267250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775182.267252:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775182.267256:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775182.267257:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775183.163252:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775183.163262:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775183.163264:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775183.163272:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775183.163277:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775183.163282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775183.163286:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775183.163289:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775183.163292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775183.291281:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775183.291286:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775183.291292:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775183.291293:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775183.291296:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775183.291297:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775183.291407:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775183.291411:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775183.291414:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775184.187278:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775184.187287:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775184.187289:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775184.187297:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775184.187303:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775184.187308:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775184.187311:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775184.187314:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775184.187319:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775184.315215:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775184.315221:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775184.315227:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775184.315228:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775184.315231:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775184.315232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775185.211255:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775185.211263:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775185.211265:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775185.211271:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775185.211275:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775185.211279:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775185.211282:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775185.211286:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775185.211289:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1775775185.211441:0:2401:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:2.0:1775775185.211489:0:2401:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1775775185.211500:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775185.211504:0:2401:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:2.0:1775775185.211509:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775185.211513:0:2401:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:2.0:1775775185.211523:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775185.211528:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:2.0:1775775185.211534:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775185.211538:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775185.211545:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3a18 00000400:00000010:2.0:1775775185.211548:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3a18. 00000100:00000001:2.0:1775775185.211555:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775185.211564:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f622e061500 x1862022710320128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775207 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775185.211600:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775185.211606:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061500 x1862022710320128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775207 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775185.211615:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775185.211618:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775775185.211631:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775185.211634:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230c00. 00000400:00000200:2.0:1775775185.211637:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775185.211641:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775185.211644:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:2.0:1775775185.211647:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:2.0:1775775185.211649:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775185.211654:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f622e061f80 x1862022710320256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775207 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775185.211664:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775185.211667:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061f80 x1862022710320256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775207 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775185.211674:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775185.211675:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775775185.211681:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775185.211684:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230000. 00000400:00000200:2.0:1775775185.211687:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775185.211690:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775185.211694:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3110 00000400:00000010:2.0:1775775185.211696:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3110. 00000100:00000001:2.0:1775775185.211699:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775185.211703:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f622e063b80 x1862022710320384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775207 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775185.211710:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775185.211713:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e063b80 x1862022710320384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775207 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775185.211718:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775185.211720:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:3.0:1775775185.339251:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775185.339256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775185.339260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775185.339263:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775185.339265:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775185.339267:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775185.339269:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775185.339271:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775185.339272:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775185.339274:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775185.339277:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775185.339278:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775185.339278:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775185.339279:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775185.339280:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775185.339281:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775185.339282:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775185.339283:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775186.235266:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775186.235274:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775186.235276:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775186.235282:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775186.235288:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775186.235294:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775186.235298:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775186.235301:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775186.235304:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775186.363218:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775186.363224:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775186.363230:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775186.363231:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775186.363234:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775186.363235:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775186.363257:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775186.363260:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775186.363263:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775187.259277:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775187.259284:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775187.259286:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775187.259291:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775187.259294:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775187.259298:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775187.259301:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775187.259303:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775187.259305:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775187.259331:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775187.259343:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775187.259346:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775187.259351:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775187.259355:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775187.259363:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12069 00000400:00000200:2.0:1775775187.259373:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775187.259379:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775187.259382:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775187.259385:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775187.259388:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775187.259390:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775187.259392:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775187.259395:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775187.259397:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775187.259400:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775187.259402:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775187.259404:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775187.259406:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775187.259408:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775187.259410:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775187.259411:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775187.259413:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775187.259416:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775187.259418:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775187.259420:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775187.259423:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775187.259426:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775187.259428:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775187.259430:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775187.259432:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775187.259439:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775775187.259442:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775187.259446:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775187.259448:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775187.259451:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775187.259454:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775187.259460:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775187.259470:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775775187.259479:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775187.259491:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710320512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775187.259505:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775187.259511:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775187.259517:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710320512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775187.259546:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775187.259550:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775187.259555:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775187.259556:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775187.259558:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775187.259562:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775187.259565:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12069 00000400:00000200:2.0:1775775187.259570:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775187.259574:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775187.259576:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775187.259580:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775187.259583:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775187.259586:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775187.259588:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775187.259591:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775187.259592:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775187.259594:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775187.259597:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775187.259599:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775187.259604:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775187.259609:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775187.259613:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775187.259617:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775187.259620:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775187.259623:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775187.259629:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775187.259635:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775187.259645:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775187.259659:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775187.259665:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775187.259670:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775187.259677:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775187.259683:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775187.259686:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775187.259693:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775775187.259698:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775187.259702:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775187.259704:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775187.259706:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775187.259710:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775187.259713:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775187.259720:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775775187.259725:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775187.259734:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710320640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775187.259744:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775187.259749:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775187.259753:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710320640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775187.259764:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775187.259770:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775187.259775:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775187.259778:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775187.259781:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775187.259786:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775187.259793:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12069 00000400:00000200:2.0:1775775187.259798:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775187.259802:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775187.259804:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775187.259807:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775187.259812:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775187.259817:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775187.259821:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775187.259823:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775187.259827:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775187.259830:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775187.259834:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775187.259839:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775187.259844:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775187.259851:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775187.259855:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775187.259857:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775187.259863:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775187.259867:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775187.259871:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775187.259874:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775187.259876:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775187.259897:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775187.259901:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775187.259905:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775187.259909:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775187.259914:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775187.259918:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775187.259923:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775775187.259928:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775187.259932:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775187.259934:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775187.259938:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775187.259941:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775187.259944:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775187.259949:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775187.259953:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775187.259962:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710320768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775187.259971:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775187.259975:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775187.259983:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710320768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775187.259998:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775187.260003:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775187.260009:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775187.260013:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775187.260020:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775187.260030:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12079) 00000800:00000200:1.1:1775775187.260040:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775187.260045:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775775187.260783:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775187.260786:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775775187.261111:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775187.261114:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00010000:00000001:3.0:1775775187.387260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775187.387267:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775187.387272:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775187.387273:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775187.387276:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775187.387277:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775188.283275:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775188.283283:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775188.283286:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775188.283291:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775188.283295:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775188.283299:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775188.283317:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775188.283321:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775188.283324:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775188.411260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775188.411266:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775188.411272:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775188.411273:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775188.411277:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775188.411279:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775188.411359:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775188.411364:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775188.411368:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775189.307255:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775189.307265:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775189.307268:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775189.307275:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775189.307280:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775189.307296:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775189.307300:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775189.307303:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775189.307307:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775189.435270:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775189.435277:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775189.435294:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775189.435295:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775189.435299:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775189.435300:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775190.331242:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775190.331249:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775190.331251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775190.331258:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775190.331262:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775190.331290:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775190.331294:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775190.331296:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775190.331299:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775190.459259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775190.459294:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775190.459300:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775190.459302:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775190.459306:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775190.459308:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775191.355240:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775191.355245:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775191.355247:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775191.355276:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775191.355279:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775191.355282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775191.355284:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775191.355286:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775191.355288:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775191.483267:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775191.483274:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775191.483275:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775191.483278:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775191.483282:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775191.483284:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775191.483285:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775191.483287:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775191.483291:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775191.483292:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775191.483295:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775191.483296:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775191.483315:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775191.483317:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775191.483319:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775192.379282:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775192.379287:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775192.379291:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775192.379296:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775192.379300:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775192.379305:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775192.379309:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775192.379311:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775192.379314:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775192.379852:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775192.379856:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1775775192.380266:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775192.380268:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1775775192.380554:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775192.380556:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00010000:00000001:3.0:1775775192.507218:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775192.507222:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775192.507244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775192.507246:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775192.507248:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775192.507249:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775193.403243:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775193.403250:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775193.403253:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775193.403259:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775193.403263:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775193.403267:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775193.403270:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775193.403272:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775193.403275:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775193.531254:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775193.531261:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775193.531267:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775193.531268:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775193.531272:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775193.531273:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775193.531327:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775193.531330:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775193.531333:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775194.427250:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775194.427255:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775194.427257:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775194.427261:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775194.427264:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775194.427267:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775194.427269:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775194.427271:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775194.427273:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775194.555261:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775194.555267:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775194.555273:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775194.555274:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775194.555278:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775194.555279:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775195.451223:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775195.451229:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775195.451230:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775195.451235:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775195.451238:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775195.451241:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775195.451243:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775195.451245:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775195.451247:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775195.579263:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775195.579268:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775195.579272:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775195.579275:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775195.579277:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775195.579279:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775195.579281:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775195.579283:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775195.579285:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775195.579286:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775195.579289:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775195.579290:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775195.579290:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775195.579291:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775195.579293:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775195.579293:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775195.579295:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775195.579296:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775196.475235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775196.475240:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775196.475242:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775196.475247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775196.475250:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775196.475252:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775196.475255:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775196.475257:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775196.475258:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775196.603229:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775196.603234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775196.603238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775196.603239:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775196.603241:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775196.603242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775196.603259:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775196.603261:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775196.603263:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775197.499236:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775197.499242:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775197.499245:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775197.499249:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775775197.499249:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775197.499254:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775775197.499256:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775197.499257:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775197.499260:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775775197.499261:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1775775197.499263:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775197.499264:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775197.499269:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775197.499270:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775197.499277:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12080 00000400:00000200:2.0:1775775197.499286:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775197.499291:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775197.499294:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775197.499297:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775197.499299:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775197.499301:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775197.499302:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775197.499305:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775197.499306:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775197.499308:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775197.499310:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775197.499312:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775197.499314:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775197.499316:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775197.499318:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775197.499319:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775197.499321:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775197.499323:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775197.499325:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775197.499326:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775197.499329:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775197.499331:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775197.499333:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775197.499335:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775197.499336:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775197.499343:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775775197.499345:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775197.499349:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775197.499350:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775197.499352:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775197.499354:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775197.499357:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775197.499363:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775775197.499368:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775197.499376:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710321280/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775197.499384:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775197.499387:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775197.499390:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710321280/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775197.499413:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775197.499415:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775197.499418:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775197.499419:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775197.499420:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775197.499423:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775197.499424:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12080 00000400:00000200:2.0:1775775197.499428:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775197.499431:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775197.499433:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775197.499436:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775197.499438:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775197.499439:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775197.499441:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775197.499445:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775197.499447:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775197.499449:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775197.499451:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775197.499453:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775197.499456:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775197.499460:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775197.499464:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775197.499469:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775197.499473:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775197.499477:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775197.499481:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775197.499484:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775197.499486:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775197.499493:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775197.499500:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775197.499512:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775197.499518:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775197.499524:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775197.499529:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775197.499538:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775775197.499544:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775197.499549:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775197.499552:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775197.499556:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775197.499559:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775197.499563:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775197.499570:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775197.499578:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775197.499588:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710321408/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775197.499605:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775197.499608:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775197.499613:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710321408/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775197.499623:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775197.499628:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775197.499632:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775197.499634:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775197.499635:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775197.499640:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775197.499645:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12080 00000400:00000200:2.0:1775775197.499650:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775197.499653:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775197.499657:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775197.499662:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775197.499666:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775197.499671:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775197.499674:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775197.499677:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775197.499680:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775197.499683:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775197.499686:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775197.499690:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775197.499693:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775197.499696:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775197.499702:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775197.499708:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775197.499712:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775197.499719:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775197.499724:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775197.499727:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775197.499732:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775197.499737:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.1:1775775197.499742:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:2.0:1775775197.499743:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775197.499746:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000100:00000001:2.0:1775775197.499747:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775197.499752:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775197.499758:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775197.499762:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775197.499768:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775775197.499772:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775197.499778:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775197.499781:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775197.499785:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775197.499788:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775197.499792:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775197.499799:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775775197.499805:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775197.499816:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710321536/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775197.499824:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775197.499828:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775197.499842:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710321536/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775197.499851:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775197.499855:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775197.499858:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775197.499865:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775197.499871:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775197.499880:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12090) 00000800:00000200:1.1:1775775197.500185:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775197.500187:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1775775197.500384:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775197.500386:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00010000:00000001:3.0:1775775197.627246:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775197.627251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775197.627256:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775197.627257:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775197.627260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775197.627261:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775198.523212:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775198.523217:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775198.523219:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775198.523223:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775198.523226:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775198.523229:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775198.523231:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775198.523233:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775198.523235:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775198.651235:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775198.651242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775198.651248:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775198.651249:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775198.651253:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775198.651254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775198.651317:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775198.651320:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775198.651323:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775199.547258:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775199.547264:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775199.547265:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775199.547271:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775199.547273:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775199.547276:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775199.547278:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775199.547280:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775199.547282:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775199.675261:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775199.675268:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775199.675273:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775199.675275:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775199.675278:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775199.675280:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775200.571212:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775200.571220:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775200.571222:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775200.571229:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775200.571233:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775200.571237:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775200.571240:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775200.571243:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775200.571245:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775200.699225:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775200.699231:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775200.699237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775200.699239:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775200.699243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775200.699244:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775201.595215:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775201.595221:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775201.595223:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775201.595227:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775201.595230:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775201.595233:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775201.595236:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775201.595237:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775201.595239:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775201.723220:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775201.723228:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775201.723239:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775201.723241:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775201.723248:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775201.723250:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775201.723252:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775201.723253:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775201.723257:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775201.723258:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775201.723261:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775201.723262:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775201.723278:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775201.723280:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775201.723282:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775202.619201:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775775202.619206:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775202.619210:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1775775202.619211:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775202.619213:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775202.619214:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775202.619219:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775775202.619219:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775202.619224:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775202.619225:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775775202.619228:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775202.619231:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1775775202.619232:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12085 00000100:00000001:3.0:1775775202.619233:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775202.619237:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775775202.619242:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775202.619248:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775202.619251:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775202.619254:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775202.619257:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775202.619260:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775202.619262:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775202.619266:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775202.619268:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775202.619271:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775202.619274:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775202.619276:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775202.619279:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775202.619282:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775202.619284:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775202.619286:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775202.619288:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775202.619291:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775202.619295:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775202.619298:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775202.619301:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775202.619304:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775202.619307:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775202.619310:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775202.619312:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775202.619320:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775775202.619324:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775202.619328:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775202.619330:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775202.619333:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775202.619336:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775202.619341:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775202.619349:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775775202.619356:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775202.619365:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710321664/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775202.619377:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775202.619381:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775202.619386:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710321664/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775202.619411:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775202.619414:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775202.619418:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775202.619420:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775202.619422:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775202.619424:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775202.619427:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12085 00000400:00000200:2.0:1775775202.619432:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775202.619436:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775202.619439:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775202.619443:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775202.619446:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775202.619448:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775202.619451:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775202.619456:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775202.619459:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775202.619463:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775202.619466:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775202.619470:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775202.619473:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775202.619481:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775202.619487:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775202.619492:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775202.619497:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775202.619503:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775202.619512:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775202.619518:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775202.619523:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775202.619529:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775202.619535:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775202.619539:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775202.619544:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775202.619550:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775202.619553:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775202.619559:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775775202.619562:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775202.619568:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775202.619569:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775202.619571:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775202.619574:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775202.619576:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775202.619581:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775775202.619586:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775202.619597:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710321792/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775202.619618:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775202.619623:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775202.619631:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710321792/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775202.619647:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775202.619651:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775202.619656:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775202.619660:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775202.619663:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775202.619668:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775202.619673:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12085 00000400:00000200:2.0:1775775202.619680:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775202.619687:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775202.619692:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775202.619700:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775202.619705:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775202.619708:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775202.619711:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775202.619713:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775202.619716:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775202.619718:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775202.619720:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775202.619723:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775202.619726:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:1.1:1775775202.619730:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775775202.619732:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775202.619735:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1775775202.619735:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000100:00000040:2.0:1775775202.619739:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775202.619742:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775202.619745:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775202.619749:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775202.619752:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775202.619754:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775202.619758:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775202.619763:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775202.619766:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775202.619770:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775202.619775:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775202.619776:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775202.619779:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775775202.619783:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775202.619788:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775202.619791:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775202.619795:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775202.619797:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775202.619799:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775202.619806:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775775202.619812:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775202.619821:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710321920/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775202.619835:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775202.619838:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775202.619843:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710321920/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775202.619852:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775202.619854:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775202.619867:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775202.619871:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775202.619876:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775202.619882:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12095) 00000800:00000200:1.1:1775775202.620124:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775202.620126:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775775202.620339:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775202.620341:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00010000:00000001:3.0:1775775202.747225:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775202.747231:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775202.747236:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775202.747238:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775202.747241:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775202.747243:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775203.643201:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775203.643207:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775203.643209:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775203.643213:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775203.643217:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775203.643220:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775203.643222:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775203.643223:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775203.643225:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775203.771212:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775203.771218:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775203.771223:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775203.771225:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775203.771229:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775203.771230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775203.771275:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775203.771278:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775203.771281:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775204.667225:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775204.667232:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775204.667234:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775204.667240:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775204.667244:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775204.667248:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775204.667250:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775204.667252:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775204.667255:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775204.795272:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775204.795278:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775204.795285:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775204.795286:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775204.795289:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775204.795290:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775205.691223:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775205.691228:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775205.691229:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775205.691234:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775205.691237:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775205.691239:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775205.691241:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775205.691243:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775205.691245:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775205.819235:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775205.819250:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775205.819259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775205.819264:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775205.819267:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775205.819272:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775205.819275:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775205.819278:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775205.819282:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775205.819284:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775205.819289:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775205.819291:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775205.819292:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775205.819294:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775205.819295:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775205.819296:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775205.819298:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775205.819300:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775206.715244:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775206.715251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775206.715253:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775206.715258:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775206.715262:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775206.715265:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775206.715268:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775206.715271:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775206.715274:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775206.843215:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775206.843220:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775206.843223:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775206.843224:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775206.843227:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775206.843228:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775206.843246:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775206.843248:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775206.843250:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775207.739231:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775207.739237:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775207.739239:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775207.739243:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775207.739246:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775207.739249:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775207.739251:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775207.739253:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775207.739255:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775207.739917:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775207.739929:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775775207.740540:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775207.740543:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775775207.740878:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775207.740881:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00010000:00000001:3.0:1775775207.867200:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775207.867205:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775207.867210:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775207.867211:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775207.867213:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775207.867214:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775208.763215:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775208.763222:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775208.763224:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775208.763228:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775208.763231:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775208.763234:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775208.763236:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775208.763238:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775208.763241:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775208.891262:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775208.891267:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775208.891271:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775208.891271:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775208.891274:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775208.891275:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775208.891353:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775208.891359:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775208.891365:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775209.787210:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775209.787216:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775209.787217:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775209.787221:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775209.787224:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775209.787227:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775209.787229:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775209.787231:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775209.787233:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775209.915230:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775209.915236:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775209.915243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775209.915244:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775209.915248:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775209.915250:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775210.811207:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775210.811212:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775210.811214:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775210.811218:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775210.811221:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775210.811224:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775210.811226:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775210.811228:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775210.811230:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775210.939223:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775210.939229:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775210.939235:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775210.939236:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775210.939239:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775210.939240:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775211.835249:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775211.835258:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775211.835260:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775211.835268:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775211.835273:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775211.835279:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775211.835282:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775211.835286:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775211.835289:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775211.963235:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775211.963242:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775211.963244:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775211.963248:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775211.963252:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775211.963255:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775211.963257:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775211.963260:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775211.963264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775211.963266:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775211.963270:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775211.963271:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775211.963292:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775211.963296:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775211.963298:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775212.859223:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775212.859230:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775212.859233:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775212.859237:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775212.859244:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775212.859248:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775212.859251:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775212.859254:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775212.859260:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775212.859822:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775212.859826:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775775212.860280:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775212.860282:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775775212.860545:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775212.860548:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00010000:00000001:3.0:1775775212.987216:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775212.987221:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775212.987227:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775212.987228:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775212.987232:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775212.987233:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775213.883249:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775213.883255:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775213.883256:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775213.883260:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775213.883264:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775213.883267:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775213.883269:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775213.883271:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775213.883273:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775214.011194:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775214.011199:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775214.011203:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775214.011204:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775214.011207:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775214.011208:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775214.011257:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775214.011260:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775214.011263:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775214.907204:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775214.907210:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775214.907212:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775214.907216:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775214.907219:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775214.907222:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775214.907224:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775214.907226:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775214.907228:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775215.035224:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775215.035229:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775215.035233:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775215.035234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775215.035243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775215.035245:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775215.931280:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775215.931288:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775215.931290:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775215.931296:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775215.931301:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775215.931307:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775215.931310:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775215.931314:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775215.931317:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775216.059247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775216.059253:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775216.059259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775216.059263:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775216.059265:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775216.059268:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775216.059270:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775216.059272:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775216.059274:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775216.059276:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775216.059279:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775216.059280:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775216.059281:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775216.059281:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775216.059282:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775216.059283:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775216.059285:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775216.059286:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775216.955209:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775216.955218:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775216.955220:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775216.955228:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775216.955232:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775216.955237:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775216.955240:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775216.955243:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775216.955246:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775217.083292:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775217.083299:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775217.083305:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775217.083306:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775217.083309:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775217.083310:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775217.083340:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775217.083343:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775217.083345:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775217.979269:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775217.979277:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775217.979282:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775217.979289:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775217.979295:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775217.979302:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775217.979307:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775217.979311:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775217.979316:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775217.979921:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775217.979925:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775775217.980309:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775217.980311:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775775217.980615:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775217.980616:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00010000:00000001:3.0:1775775218.107261:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775218.107267:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775218.107273:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775218.107274:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775218.107277:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775218.107278:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775219.003230:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775219.003249:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775219.003251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775219.003259:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775219.003264:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775219.003269:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775219.003272:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775219.003275:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775219.003278:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775219.131261:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775219.131268:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775219.131273:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775219.131274:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775219.131277:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775219.131278:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775219.131330:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775219.131334:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775219.131337:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775220.027253:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775220.027264:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775220.027266:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775220.027274:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775220.027280:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775220.027285:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775220.027289:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775220.027292:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775220.027295:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775220.155236:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775220.155244:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775220.155252:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775220.155254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775220.155259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775220.155260:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775221.051258:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775221.051266:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775221.051268:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775221.051275:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775221.051278:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775221.051281:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775221.051283:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775221.051285:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775221.051288:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775221.179262:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775221.179270:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775221.179275:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775221.179277:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775221.179280:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775221.179281:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775222.075212:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775222.075219:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775222.075221:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775222.075226:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775222.075229:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775222.075233:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775222.075236:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775222.075238:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775222.075240:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775222.203222:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775222.203231:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775222.203233:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775222.203237:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775222.203246:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775222.203249:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775222.203251:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775222.203253:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775222.203257:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775222.203258:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775222.203261:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775222.203263:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775222.203287:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775222.203289:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775222.203292:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775223.099196:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775223.099204:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775223.099206:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775223.099212:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775223.099215:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775223.099218:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775223.099221:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775223.099223:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775223.099226:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775223.099239:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775223.099244:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775223.099246:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775223.099249:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775223.099253:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775223.099260:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12105 00000400:00000200:3.0:1775775223.099271:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775223.099278:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775223.099281:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775223.099285:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775223.099289:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775223.099292:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775223.099294:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775223.099300:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775223.099302:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775223.099306:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775223.099309:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775223.099312:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775223.099315:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775223.099319:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775223.099321:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775223.099323:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775223.099325:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775223.099328:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775223.099332:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775223.099335:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775223.099339:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775223.099342:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775223.099346:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775223.099348:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775223.099351:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775223.099359:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73100. 00000020:00000040:3.0:1775775223.099363:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775223.099368:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775223.099371:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775223.099374:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775223.099378:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775223.099382:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775223.099391:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775775223.099399:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775223.099408:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73100 x1862022710323200/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775223.099418:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775223.099422:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775223.099427:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f67e6552] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73100 x1862022710323200/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775223.099455:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775223.099459:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775223.099463:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775223.099465:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775223.099466:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775223.099470:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775223.099473:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12105 00000400:00000200:3.0:1775775223.099478:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775223.099483:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775223.099486:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775223.099488:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775223.099490:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775223.099492:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775223.099494:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775223.099496:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775223.099498:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775223.099499:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775223.099501:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775223.099504:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775223.099506:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775223.099511:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775223.099515:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775223.099516:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775223.099518:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775223.099521:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775223.099525:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775223.099530:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775223.099535:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775223.099541:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775223.099546:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775223.099550:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775223.099556:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775223.099561:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775223.099564:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775223.099568:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775775223.099572:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775223.099575:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775223.099577:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775223.099579:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775223.099582:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775223.099584:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775223.099589:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000001:3.0:1775775223.099593:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775223.099599:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710323328/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775223.099610:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775223.099612:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775223.099616:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710323328/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775223.099628:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775223.099630:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775223.099635:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775223.099638:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775223.099640:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775223.099643:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775223.099647:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12105 00000400:00000200:3.0:1775775223.099650:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775223.099652:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775223.099654:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775223.099659:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775223.099660:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775223.099662:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775223.099663:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775223.099665:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775223.099666:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775223.099668:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775223.099669:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775223.099670:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775223.099671:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775223.099673:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775223.099674:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775223.099676:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775223.099679:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775223.099681:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775223.099684:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775223.099689:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775223.099691:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775223.099693:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775223.099709:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775223.099713:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775223.099715:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775223.099719:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775223.099722:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775223.099728:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000020:00000040:3.0:1775775223.099732:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775223.099737:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775223.099740:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775223.099741:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775223.099743:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775223.099745:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775223.099751:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000001:3.0:1775775223.099756:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775223.099762:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710323456/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775223.099770:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775223.099772:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775223.099776:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710323456/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775223.099785:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775223.099787:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775223.099791:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775223.099795:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775223.099798:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775223.099802:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12115) 00000800:00000200:0.1:1775775223.099886:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775223.099890:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775775223.100408:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775223.100411:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775775223.100752:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775223.100755:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00010000:00000001:3.0:1775775223.227278:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775223.227286:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775223.227294:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775223.227296:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775223.227300:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775223.227301:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775224.123182:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775224.123188:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775224.123189:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775224.123195:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775224.123198:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775224.123202:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775224.123204:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775224.123206:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775224.123209:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775224.251254:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775224.251262:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775224.251268:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775224.251271:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775224.251274:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775224.251275:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775224.251346:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775224.251350:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775224.251353:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775225.147248:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775225.147256:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775225.147258:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775225.147264:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775225.147268:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775225.147272:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775225.147274:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775225.147277:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775225.147289:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775225.275213:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775225.275218:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775225.275222:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775225.275223:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775225.275225:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775225.275226:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775226.171290:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775226.171299:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775226.171302:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775226.171310:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775226.171315:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775226.171320:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775226.171323:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775226.171327:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775226.171331:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775226.299283:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775226.299290:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775226.299295:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775226.299298:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775226.299300:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775226.299304:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775226.299306:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775226.299309:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775226.299311:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775226.299312:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775226.299315:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775226.299317:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775226.299318:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775226.299319:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775226.299321:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775226.299322:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775226.299323:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775226.299324:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775227.195267:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775227.195275:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775227.195276:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775227.195282:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775227.195286:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775227.195290:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775227.195293:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775227.195296:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775227.195298:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775227.323213:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775227.323218:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775227.323223:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775227.323224:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775227.323227:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775227.323228:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775227.323246:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775227.323249:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775227.323252:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775228.219233:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775775228.219233:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775228.219240:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1775775228.219241:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775228.219244:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775775228.219244:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775228.219250:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775775228.219250:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775228.219255:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775228.219256:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775775228.219258:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775228.219261:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775228.219263:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775775228.219264:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12110 00000400:00000200:2.0:1775775228.219272:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775775228.219274:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775775228.219281:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775228.219283:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775228.219286:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775228.219293:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775228.219296:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775228.219298:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775228.219300:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775228.219302:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775228.219304:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775228.219307:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775228.219308:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775228.219310:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775228.219312:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775228.219314:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775228.219315:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775228.219317:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775228.219319:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775228.219321:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775228.219323:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775228.219325:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775228.219327:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775228.219329:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775228.219330:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775228.219332:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775228.219338:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775775228.219341:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775228.219344:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775228.219345:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775228.219347:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775228.219349:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775228.219353:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775228.219358:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775775228.219364:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775228.219370:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710323584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775228.219378:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775228.219382:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775228.219385:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710323584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775228.219402:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775228.219404:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775228.219406:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775228.219407:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775228.219409:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775228.219410:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775228.219413:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12110 00000400:00000200:2.0:1775775228.219417:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775228.219420:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775228.219422:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775228.219425:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775228.219428:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775228.219431:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775228.219433:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775228.219436:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775228.219437:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775228.219439:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775228.219442:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775228.219444:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775228.219449:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775228.219455:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775228.219459:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775228.219465:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775228.219471:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775228.219476:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775228.219479:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775228.219483:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775228.219485:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775228.219490:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775228.219494:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775228.219495:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775228.219499:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775228.219504:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775228.219506:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775228.219509:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775775228.219512:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775228.219515:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775228.219517:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775228.219518:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775228.219519:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775228.219520:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775228.219523:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775228.219525:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775228.219529:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710323712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775228.219538:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775228.219540:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775228.219543:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710323712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775228.219557:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775228.219562:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775228.219567:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775228.219569:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775228.219574:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775228.219577:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775228.219582:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12110 00000400:00000200:2.0:1775775228.219592:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775228.219601:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775228.219619:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775228.219628:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775228.219634:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775228.219640:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775228.219644:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775228.219648:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775228.219651:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775228.219655:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775228.219659:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775228.219661:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775228.219664:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775228.219671:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775228.219673:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775228.219677:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775228.219682:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775228.219687:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775228.219694:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775228.219699:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775228.219704:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775228.219710:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775228.219716:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775228.219719:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775228.219723:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775228.219729:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775228.219734:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775775228.219741:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000010:2.0:1775775228.219742:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000800:00000200:0.1:1775775228.219745:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000020:00000040:2.0:1775775228.219747:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775228.219753:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775228.219757:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775228.219760:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775228.219764:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775228.219768:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775228.219774:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775775228.219780:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775228.219788:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710323840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775228.219800:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775228.219806:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775228.219816:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710323840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775228.219838:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775228.219845:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775228.219854:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775228.219858:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775228.219866:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775228.219873:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12120) 00000800:00000200:0.1:1775775228.220090:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775228.220092:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775775228.220325:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775228.220326:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00010000:00000001:3.0:1775775228.347205:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775228.347212:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775228.347218:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775228.347219:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775228.347224:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775228.347225:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775229.243217:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775229.243224:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775229.243226:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775229.243232:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775229.243236:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775229.243240:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775229.243243:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775229.243245:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775229.243248:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775229.371241:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775229.371246:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775229.371252:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775229.371254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775229.371258:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775229.371259:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775229.371320:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775229.371324:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775229.371327:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775230.267213:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775230.267220:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775230.267222:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775230.267226:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775230.267229:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775230.267232:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775230.267234:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775230.267236:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775230.267238:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775230.395221:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775230.395225:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775230.395230:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775230.395230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775230.395233:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775230.395234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775231.291236:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775231.291243:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775231.291245:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775231.291249:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775231.291252:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775231.291255:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775231.291258:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775231.291261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775231.291263:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775231.419232:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775231.419239:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775231.419241:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775231.419245:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775231.419249:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775231.419252:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775231.419254:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775231.419257:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775231.419261:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775231.419262:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775231.419266:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775231.419268:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775232.315217:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775232.315222:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775232.315224:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775232.315228:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775232.315231:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775232.315234:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775232.315237:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775232.315238:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775232.315240:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775232.443268:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775232.443274:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775232.443282:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775232.443283:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775232.443287:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775232.443289:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775232.443315:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775232.443317:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775232.443320:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775233.339226:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775775233.339230:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775233.339236:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1775775233.339237:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775233.339239:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775233.339241:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1775775233.339243:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:3.0:1775775233.339246:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775233.339247:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775233.339253:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775233.339254:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775775233.339256:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775233.339260:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775233.339262:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775233.339265:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12115 00000400:00000200:3.0:1775775233.339276:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775233.339283:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775233.339287:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775233.339291:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775233.339295:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775233.339298:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775233.339300:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775233.339304:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775233.339306:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775233.339309:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775233.339312:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775233.339314:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775233.339317:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775233.339320:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775233.339322:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775233.339324:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775233.339327:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775233.339330:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775233.339333:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775233.339336:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775233.339339:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775233.339342:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775233.339345:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775233.339347:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775233.339349:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775233.339358:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775775233.339362:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775233.339366:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775233.339368:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775233.339372:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775233.339374:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775233.339377:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775233.339385:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000100:00000001:3.0:1775775233.339392:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775233.339403:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710323968/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775233.339414:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775233.339418:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775233.339424:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710323968/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775233.339447:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775233.339450:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775233.339453:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775233.339456:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775233.339458:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775233.339461:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775233.339465:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12115 00000400:00000200:3.0:1775775233.339470:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775233.339474:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775233.339477:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775233.339481:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775233.339484:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775233.339491:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775233.339499:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775233.339507:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775233.339532:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775233.339538:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775233.339542:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775233.339548:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775233.339570:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775233.339584:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775233.339587:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775233.339595:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775233.339601:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775233.339609:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775233.339617:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775233.339632:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775233.339643:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775233.339653:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775233.339663:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775233.339670:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775233.339677:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775233.339686:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775233.339689:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775233.339695:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775775233.339699:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775233.339711:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775233.339715:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775233.339727:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775233.339730:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775233.339736:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775233.339746:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775775233.339754:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775233.339766:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710324096/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775233.339795:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775233.339804:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775775233.339816:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775775233.339822:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710324096/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1775775233.339823:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 11 secs later) 00000100:00000001:3.0:1775775233.339851:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775233.339855:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775233.339860:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775233.339868:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775233.339871:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775233.339881:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775233.339890:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12115 00000400:00000200:3.0:1775775233.339909:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775233.339924:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775233.339932:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775233.339944:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775233.339954:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775233.339962:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775233.339969:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775233.339973:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775233.339976:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775233.339982:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775233.339990:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775233.339997:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775233.340006:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775233.340011:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775233.340020:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775233.340022:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775233.340024:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775233.340027:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775233.340036:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775233.340042:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775233.340051:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775233.340055:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775233.340058:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775233.340068:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775233.340070:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775233.340078:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775233.340090:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775233.340104:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775775233.340109:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775233.340117:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775233.340119:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775233.340125:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775233.340127:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775233.340153:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775233.340158:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000100:00000001:3.0:1775775233.340162:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775233.340167:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710324224/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775233.340174:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775233.340176:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775233.340186:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710324224/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775233.340219:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775233.340230:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775233.340245:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775233.340257:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775233.340265:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775233.340275:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12126) 00000800:00000200:0.1:1775775233.340513:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775233.340529:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1775775233.340777:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775233.340780:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 11 secs later) 00010000:00000001:3.0:1775775233.467221:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775233.467226:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775233.467230:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775233.467231:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775233.467233:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775233.467234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775234.363199:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775234.363206:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775234.363208:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775234.363215:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775234.363219:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775234.363223:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775234.363226:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775234.363228:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775234.363231:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775234.491242:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775234.491247:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775234.491251:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775234.491252:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775234.491256:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775234.491257:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775234.491312:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775234.491316:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775234.491318:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775235.387220:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775235.387225:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775235.387226:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775235.387230:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775235.387233:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775235.387236:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775235.387239:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775235.387241:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775235.387243:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775235.515308:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775235.515315:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775235.515321:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775235.515323:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775235.515327:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775235.515328:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775236.411294:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775236.411302:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775236.411303:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775236.411309:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775236.411312:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775236.411316:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775236.411319:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775236.411321:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775236.411323:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775236.539249:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775236.539256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775236.539263:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775236.539267:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775236.539269:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775236.539273:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775236.539276:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775236.539278:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775236.539280:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775236.539283:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775236.539287:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775236.539288:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775236.539290:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775236.539292:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775236.539293:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775236.539295:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775236.539296:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775236.539299:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775237.435195:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775237.435200:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775237.435202:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775237.435206:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775237.435209:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775237.435212:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775237.435216:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775237.435218:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775237.435220:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775237.563221:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775237.563228:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775237.563233:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775237.563234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775237.563238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775237.563240:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775237.563261:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775237.563263:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775237.563266:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775238.459203:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775238.459209:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775238.459211:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775238.459213:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775775238.459215:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775238.459220:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775775238.459222:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775238.459223:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775238.459227:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775238.459228:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775775238.459229:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775775238.459233:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775238.459239:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775775238.459240:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775238.459246:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12121 00000400:00000200:3.0:1775775238.459254:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775238.459260:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775238.459263:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775238.459265:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775238.459267:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775238.459269:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775238.459271:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775238.459274:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775238.459275:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775238.459277:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775238.459280:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775238.459281:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775238.459283:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775238.459285:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775238.459287:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775238.459288:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775238.459289:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775238.459291:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775238.459293:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775238.459295:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775238.459297:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775238.459299:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775238.459301:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775238.459303:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775238.459306:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775238.459313:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775775238.459315:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775238.459326:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775238.459328:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775238.459330:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775238.459332:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775238.459336:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775238.459343:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000100:00000001:3.0:1775775238.459348:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775238.459355:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710324352/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775238.459363:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775238.459367:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775238.459370:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710324352/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775238.459387:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775238.459390:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775238.459392:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775238.459393:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775238.459394:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775238.459396:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775238.459398:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12121 00000400:00000200:3.0:1775775238.459400:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775238.459402:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775238.459404:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775238.459407:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775238.459409:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775238.459410:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775238.459412:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775238.459414:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775238.459416:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775238.459417:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775238.459419:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775238.459421:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775238.459422:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775238.459425:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775238.459450:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775238.459455:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775238.459459:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775238.459464:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775238.459469:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775238.459473:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775238.459477:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775238.459482:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775238.459487:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775238.459491:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775238.459497:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775238.459501:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775238.459505:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775238.459512:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775775238.459514:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775238.459516:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775238.459519:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775238.459522:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775238.459529:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775238.459532:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775238.459540:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000001:3.0:1775775238.459544:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775238.459552:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710324480/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775238.459568:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775238.459572:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775238.459579:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710324480/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775238.459593:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775238.459596:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775238.459603:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775238.459607:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775238.459608:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775238.459610:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775238.459615:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12121 00000400:00000200:3.0:1775775238.459621:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775238.459628:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775238.459631:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775238.459634:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775238.459637:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775238.459642:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775238.459647:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775238.459651:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775238.459654:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775238.459656:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775238.459659:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775238.459664:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775238.459667:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775238.459670:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775238.459675:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775238.459681:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775238.459685:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775238.459687:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775238.459693:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775238.459696:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775238.459699:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775238.459702:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775238.459704:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775238.459708:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775238.459710:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775238.459714:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775238.459715:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775238.459721:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000800:00000200:0.1:1775775238.459722:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000040:3.0:1775775238.459724:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000800:00000200:0.1:1775775238.459728:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 02000000:00000001:3.0:1775775238.459729:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775238.459731:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775238.459734:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775238.459737:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775238.459739:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775238.459743:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000001:3.0:1775775238.459747:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775238.459755:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710324608/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775238.459764:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775238.459769:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775238.459776:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710324608/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775238.459789:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775238.459794:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775238.459800:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775238.459804:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775238.459812:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775238.459819:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12131) 00000800:00000200:0.1:1775775238.460048:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775238.460049:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1775775238.460302:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775238.460304:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00010000:00000001:3.0:1775775238.587235:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775238.587240:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775238.587246:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775238.587247:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775238.587250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775238.587251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775239.483197:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775239.483203:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775239.483205:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775239.483209:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775239.483212:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775239.483215:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775239.483218:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775239.483220:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775239.483223:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775239.611219:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775239.611225:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775239.611231:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775239.611232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775239.611235:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775239.611236:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775240.507256:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775240.507262:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775240.507263:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775240.507268:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775240.507270:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775240.507274:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775240.507276:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775240.507278:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775240.507280:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775240.635217:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775240.635222:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775240.635226:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775240.635226:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775240.635230:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775240.635230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775241.659259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775241.659266:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775241.659269:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775241.659273:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775241.659277:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775241.659280:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775241.659282:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775241.659284:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775241.659288:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775241.659290:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775241.659294:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775241.659295:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775242.683243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775242.683248:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775242.683252:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775242.683253:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775242.683256:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775242.683257:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775242.683275:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775242.683277:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775242.683279:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1775775243.579767:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775243.579772:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775775243.580241:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775243.580243:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775775243.580567:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775243.580570:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00010000:00000001:3.0:1775775243.707245:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775243.707254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775243.707259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775243.707259:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775243.707262:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775243.707263:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775244.731223:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775244.731229:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775244.731235:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775244.731237:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775244.731240:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775244.731241:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775245.755278:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775245.755286:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775245.755295:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775245.755297:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775245.755316:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775245.755318:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775246.779277:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775246.779310:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775246.779316:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775246.779319:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775246.779320:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775246.779323:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775246.779325:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775246.779327:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775246.779328:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775246.779329:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775246.779333:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775246.779334:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775246.779335:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775246.779335:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775246.779336:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775246.779337:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775246.779338:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775246.779339:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775247.803230:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775247.803234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775247.803239:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775247.803241:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775247.803243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775247.803244:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775247.803263:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775247.803289:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775247.803292:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775248.699286:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775248.699294:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775248.699299:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775248.699303:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775248.699308:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775248.699316:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12131 00000400:00000200:2.0:1775775248.699325:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775248.699341:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775248.699348:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775248.699352:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775248.699356:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775248.699358:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775248.699361:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775248.699365:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775248.699367:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775248.699370:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775248.699373:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775248.699374:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775248.699376:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775248.699379:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775248.699380:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775248.699381:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775248.699383:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775248.699385:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775248.699387:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775248.699389:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775248.699391:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775248.699393:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775248.699395:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775248.699397:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775248.699399:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775248.699406:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775775248.699408:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775248.699411:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775248.699412:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775248.699414:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775248.699417:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775248.699420:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775248.699427:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775775248.699432:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775248.699439:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710325120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775248.699448:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775248.699451:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775248.699454:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710325120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775248.699470:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775248.699473:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775248.699477:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775248.699478:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775248.699480:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775248.699483:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775248.699486:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12131 00000400:00000200:2.0:1775775248.699491:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775248.699494:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775248.699496:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775248.699499:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775248.699502:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775248.699504:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775248.699506:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775248.699508:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775248.699510:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775248.699512:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775248.699514:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775248.699516:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775248.699519:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775248.699522:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775248.699525:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775248.699527:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775248.699529:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775248.699532:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775248.699535:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775248.699537:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775248.699539:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775248.699543:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775248.699545:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775248.699547:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775248.699551:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775248.699554:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775248.699556:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775248.699561:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775775248.699563:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775248.699566:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775248.699567:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775248.699569:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775248.699570:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775248.699572:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775248.699576:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775775248.699579:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775248.699584:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710325248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775248.699592:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775248.699595:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775248.699599:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710325248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775248.699608:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775248.699610:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775248.699614:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775248.699616:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775248.699618:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775248.699620:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775248.699623:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12131 00000400:00000200:2.0:1775775248.699626:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775248.699629:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775248.699631:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775248.699633:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775248.699636:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775248.699638:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775248.699640:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775248.699642:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775248.699644:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775248.699646:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775248.699647:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775248.699649:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775248.699651:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775248.699654:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775248.699656:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775248.699658:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775248.699660:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775248.699662:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775248.699665:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775248.699667:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775248.699668:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775248.699671:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775248.699672:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775248.699674:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775248.699676:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775248.699679:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775248.699679:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:2.0:1775775248.699681:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775248.699684:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775775248.699687:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775248.699689:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775248.699690:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775248.699692:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775248.699693:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775248.699695:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775248.699698:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775248.699701:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775248.699706:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710325376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775248.699713:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775248.699715:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775248.699719:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710325376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775248.699728:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775248.699731:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775248.699735:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775248.699738:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775248.699742:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775248.699746:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12141) 00000800:00000200:1.1:1775775248.699870:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775248.700020:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00010000:00000001:3.0:1775775248.827216:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775248.827221:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775248.827224:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775248.827225:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775248.827228:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775248.827229:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775249.851223:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775249.851252:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775249.851258:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775249.851259:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775249.851262:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775249.851263:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775250.875301:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775250.875308:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775250.875313:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775250.875314:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775250.875318:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775250.875319:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:3.0:1775775251.771526:0:2402:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:3.0:1775775251.771586:0:2402:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1775775251.771601:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775251.771607:0:2402:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:3.0:1775775251.771614:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775251.771619:0:2402:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:3.0:1775775251.771627:0:2402:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775775251.771632:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:3.0:1775775251.771638:0:2402:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775775251.771644:0:2402:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775775251.771660:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3000 00000400:00000010:3.0:1775775251.771664:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3000. 00000100:00000001:3.0:1775775251.771672:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775775251.771684:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6233449500 x1862022710325120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775273 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775251.771720:0:2402:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775775251.771726:0:2402:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449500 x1862022710325120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775273 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775251.771736:0:2402:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775251.771738:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775775251.771751:0:2402:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775775251.771755:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230400. 00000400:00000200:3.0:1775775251.771759:0:2402:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775775251.771764:0:2402:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775775251.771768:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:3.0:1775775251.771771:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:3.0:1775775251.771775:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775775251.771780:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f623344bb80 x1862022710325248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775273 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775251.771790:0:2402:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775775251.771796:0:2402:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344bb80 x1862022710325248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775273 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775251.771808:0:2402:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775251.771812:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775775251.771824:0:2402:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775775251.771828:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230600. 00000400:00000200:3.0:1775775251.771831:0:2402:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775775251.771841:0:2402:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775775251.771849:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f34c8 00000400:00000010:3.0:1775775251.771855:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f34c8. 00000100:00000001:3.0:1775775251.771862:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775775251.771871:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6233449c00 x1862022710325376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775273 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775251.771916:0:2402:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775775251.771925:0:2402:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449c00 x1862022710325376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775273 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775251.771971:0:2402:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775251.771974:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:3.0:1775775251.899291:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775251.899301:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775251.899303:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775251.899307:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775251.899312:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775251.899314:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775251.899316:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775251.899318:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775251.899321:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775251.899322:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775251.899325:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775251.899326:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775252.923273:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775252.923279:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775252.923284:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775252.923291:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775252.923295:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775252.923296:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775252.923321:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775252.923324:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775252.923327:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775253.819217:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775253.819225:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775253.819227:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775253.819230:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775253.819234:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775253.819239:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12136 00000400:00000200:2.0:1775775253.819248:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775253.819253:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775253.819256:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775253.819259:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775253.819262:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775253.819273:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775253.819275:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775253.819279:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775253.819280:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775253.819283:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775253.819286:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775253.819287:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775253.819290:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775253.819292:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775253.819294:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775253.819295:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775253.819296:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775253.819298:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775253.819301:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775253.819303:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775253.819306:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775253.819308:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775253.819312:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775253.819314:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775253.819316:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775253.819323:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775775253.819327:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775253.819332:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775253.819333:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775253.819335:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775253.819337:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775253.819341:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775253.819347:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775253.819353:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775253.819364:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710325504/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775253.819377:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775253.819382:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775253.819389:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710325504/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775253.819417:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775253.819420:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775253.819423:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775253.819425:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775253.819427:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775253.819430:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775253.819433:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12136 00000400:00000200:2.0:1775775253.819437:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775253.819440:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775253.819443:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775253.819446:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775253.819449:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775253.819451:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775253.819454:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775253.819457:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775253.819458:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775253.819460:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775253.819463:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775253.819465:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775253.819467:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775253.819471:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775253.819475:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775253.819481:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775253.819485:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775253.819489:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775253.819496:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775253.819504:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775253.819510:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775253.819519:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775253.819544:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775253.819549:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775253.819555:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775253.819560:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775253.819564:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775253.819572:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775775253.819578:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775253.819582:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775253.819584:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775253.819589:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775253.819592:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775253.819595:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775253.819601:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775775253.819607:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775253.819616:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710325632/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775253.819628:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775253.819633:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775253.819642:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710325632/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775253.819663:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775253.819667:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775253.819672:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775253.819676:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775253.819680:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775253.819685:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775253.819690:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12136 00000400:00000200:2.0:1775775253.819699:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775253.819707:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775253.819711:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775253.819717:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775253.819721:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775253.819727:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775253.819732:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775253.819737:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775253.819741:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775253.819745:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775253.819748:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775253.819752:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775253.819759:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775253.819768:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775253.819772:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775253.819775:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775253.819780:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775253.819785:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775253.819790:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775253.819796:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775253.819801:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775253.819806:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775253.819810:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775253.819815:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775253.819817:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775253.819821:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775253.819824:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775253.819829:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775775253.819835:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775253.819841:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775253.819845:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775253.819849:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:2.0:1775775253.819852:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775253.819855:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:1.1:1775775253.819857:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 02000000:00000001:2.0:1775775253.819860:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775253.819870:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775775253.819876:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775253.819885:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710325760/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775253.819904:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775253.819909:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775253.819916:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710325760/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775253.819931:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775253.819937:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775253.819946:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775253.819951:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775253.819955:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775253.819962:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12146) 00000800:00000200:1.1:1775775253.820443:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775253.820447:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775775253.820770:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775253.820772:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00010000:00000001:3.0:1775775253.947251:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775253.947256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775253.947260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775253.947261:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775253.947264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775253.947265:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775254.971287:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775254.971296:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775254.971304:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775254.971306:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775254.971311:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775254.971312:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775255.995245:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775255.995250:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775255.995254:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775255.995255:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775255.995258:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775255.995259:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775257.019226:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775257.019232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775257.019237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775257.019241:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775257.019243:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775257.019246:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775257.019249:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775257.019251:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775257.019253:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775257.019255:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775257.019259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775257.019260:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775257.019262:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775257.019263:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775257.019265:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775257.019266:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775257.019267:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775257.019269:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775258.043220:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775258.043225:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775258.043229:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775258.043230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775258.043232:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775258.043233:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775258.043254:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775258.043255:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775258.043257:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775258.939215:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775258.939224:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775258.939229:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775258.939231:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775258.939235:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775258.939242:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12141 00000400:00000200:2.0:1775775258.939253:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775258.939259:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775258.939261:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775258.939265:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775258.939268:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775258.939270:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775258.939272:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775258.939275:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775258.939276:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775258.939278:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775258.939280:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775258.939282:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775258.939283:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775258.939286:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775258.939287:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775258.939288:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775258.939289:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775258.939291:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775258.939294:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775258.939295:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775258.939297:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775258.939299:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775258.939301:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775258.939303:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775258.939304:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775258.939311:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775775258.939314:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775258.939317:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775258.939318:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775258.939320:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775258.939322:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775258.939325:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775258.939331:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775775258.939336:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775258.939343:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710325888/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775258.939351:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775258.939354:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775258.939358:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710325888/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775258.939382:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775258.939385:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775258.939387:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775258.939389:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775258.939390:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775258.939392:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775258.939393:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12141 00000400:00000200:2.0:1775775258.939396:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775258.939398:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775258.939400:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775258.939402:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775258.939404:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775258.939406:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775258.939408:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775258.939410:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775258.939411:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775258.939412:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775258.939414:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775258.939415:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775258.939416:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775258.939419:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775258.939420:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775258.939422:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775258.939424:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775258.939432:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775258.939435:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775258.939439:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775258.939440:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775258.939445:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775258.939448:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775258.939453:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775258.939458:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775258.939461:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775258.939464:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775258.939470:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775775258.939473:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775258.939476:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775258.939478:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775258.939481:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775258.939484:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775258.939485:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775258.939490:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775775258.939494:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775258.939502:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710326016/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775258.939514:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775258.939518:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775258.939521:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710326016/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775258.939533:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775258.939535:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775258.939541:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775258.939543:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775258.939545:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775258.939548:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775258.939551:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12141 00000400:00000200:2.0:1775775258.939556:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775258.939559:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775258.939563:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775258.939569:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775258.939572:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775258.939575:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775258.939580:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775258.939585:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775258.939587:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775258.939590:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775258.939593:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775258.939595:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775258.939598:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775258.939603:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775258.939607:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775258.939611:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775258.939613:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775258.939616:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775258.939621:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775258.939622:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775258.939625:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775258.939628:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775258.939630:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775258.939633:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775258.939635:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775258.939638:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775258.939639:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775258.939644:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775775258.939648:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775258.939650:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775258.939653:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775258.939657:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775258.939659:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775258.939661:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775258.939671:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775258.939675:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775258.939683:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710326144/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775258.939696:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775258.939698:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775258.939704:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710326144/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775258.939719:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775258.939723:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775775258.939723:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775775258.939730:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775775258.939730:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000100:00000040:2.0:1775775258.939735:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775258.939742:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775258.939747:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12151) 00000800:00000200:1.1:1775775258.940103:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775258.940106:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775775258.940343:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775258.940345:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00010000:00000001:3.0:1775775259.067213:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775259.067219:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775259.067226:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775259.067227:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775259.067231:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775259.067232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775775259.771280:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775775259.771290:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775775259.771296:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00010000:00000001:3.0:1775775260.091225:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775260.091230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775260.091234:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775260.091235:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775260.091238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775260.091239:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775261.115245:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775261.115250:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775261.115257:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775261.115258:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775261.115262:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775261.115264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775262.139222:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775262.139231:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775262.139234:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775262.139242:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775262.139248:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775262.139251:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775262.139253:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775262.139255:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775262.139259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775262.139261:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775262.139264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775262.139266:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775263.163239:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775263.163245:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775263.163250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775263.163251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775263.163254:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775263.163255:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775263.163274:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775263.163276:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775263.163278:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775264.059247:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775264.059254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775264.059258:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775264.059264:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775264.059268:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775264.059273:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775264.059279:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775264.059282:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775264.059287:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775264.059819:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775264.059823:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775775264.060311:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775264.060313:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775775264.060546:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775264.060549:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00010000:00000001:3.0:1775775264.187197:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775264.187201:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775264.187205:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775264.187206:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775264.187209:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775264.187209:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775265.083221:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775265.083229:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775265.083231:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775265.083238:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775265.083242:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775265.083245:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775265.083248:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775265.083251:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775265.083254:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775265.211256:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775265.211261:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775265.211265:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775265.211266:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775265.211269:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775265.211270:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775266.107276:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775266.107280:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775266.107281:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775266.107285:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775266.107288:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775266.107291:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775266.107293:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775266.107294:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775266.107297:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775266.235207:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775266.235214:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775266.235219:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775266.235220:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775266.235222:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775266.235223:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775267.131268:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775267.131275:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775267.131277:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775267.131283:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775267.131286:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775267.131290:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775267.131292:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775267.131294:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775267.131296:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775267.259280:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775267.259286:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775267.259290:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775267.259294:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775267.259296:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775267.259299:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775267.259302:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775267.259303:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775267.259305:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775267.259306:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775267.259309:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775267.259310:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775267.259310:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775267.259311:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775267.259312:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775267.259313:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775267.259314:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775267.259315:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775268.155270:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775268.155277:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775268.155279:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775268.155286:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775268.155290:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775268.155294:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775268.155297:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775268.155300:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775268.155303:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775268.283268:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775268.283276:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775268.283283:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775268.283285:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775268.283289:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775268.283290:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775268.283318:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775268.283322:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775268.283325:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775269.179239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775269.179244:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775269.179246:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775269.179253:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775269.179258:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775269.179262:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775269.179265:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775269.179268:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775269.179271:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775269.179778:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775269.179781:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775775269.180250:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775269.180252:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775775269.180622:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775269.180625:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00010000:00000001:3.0:1775775269.307244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775269.307249:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775269.307255:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775269.307256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775269.307260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775269.307261:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775270.203268:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775270.203278:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775270.203280:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775270.203287:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775270.203292:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775270.203296:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775270.203300:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775270.203303:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775270.203306:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775270.331223:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775270.331227:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775270.331232:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775270.331234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775270.331237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775270.331239:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775271.227262:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775271.227272:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775271.227274:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775271.227281:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775271.227286:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775271.227290:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775271.227293:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775271.227296:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775271.227299:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775271.355269:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775271.355277:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775271.355279:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775271.355282:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775271.355287:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775271.355289:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775271.355290:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775271.355292:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775271.355295:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775271.355296:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775271.355298:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775271.355299:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775272.379276:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775272.379284:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775272.379291:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775272.379292:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775272.379296:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775272.379298:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775273.403244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775273.403250:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775273.403256:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775273.403257:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775273.403261:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775273.403262:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775273.403287:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775273.403290:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775273.403293:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775274.299212:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775274.299220:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775274.299222:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775274.299226:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775274.299231:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775274.299239:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12156 00000400:00000200:3.0:1775775274.299251:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775274.299257:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775274.299260:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775274.299264:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775274.299268:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775274.299271:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775274.299274:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775274.299277:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775274.299279:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775274.299282:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775274.299284:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775274.299287:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775274.299289:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775274.299292:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775274.299294:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775274.299296:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775274.299298:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775274.299301:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775274.299305:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775274.299308:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775274.299311:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775274.299314:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775274.299317:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775274.299319:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775274.299322:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775274.299331:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775775274.299334:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775274.299338:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775274.299340:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775274.299343:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775274.299345:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775274.299349:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775274.299358:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775775274.299364:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775274.299372:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710327040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775274.299382:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775274.299386:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775274.299391:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710327040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775274.299416:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775274.299420:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775274.299423:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775274.299425:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775274.299426:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775274.299428:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775274.299431:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12156 00000400:00000200:3.0:1775775274.299443:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775274.299447:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775274.299450:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775274.299456:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775274.299460:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775274.299465:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775274.299470:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775274.299475:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775274.299477:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775274.299481:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775274.299485:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775274.299489:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775274.299494:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775274.299500:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775274.299504:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775274.299508:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775274.299512:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775274.299517:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775274.299521:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775274.299527:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775274.299531:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775274.299538:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775274.299542:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775274.299545:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775274.299550:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775274.299556:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775274.299560:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775274.299567:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775775274.299572:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775274.299576:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775274.299579:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775274.299582:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775274.299584:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775274.299588:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775274.299594:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000001:3.0:1775775274.299600:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775274.299609:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710327168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775274.299629:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775274.299634:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775274.299643:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710327168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775274.299660:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775274.299665:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775274.299671:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775274.299673:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:1.1:1775775274.299674:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00080000:3.0:1775775274.299676:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.1:1775775274.299678:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000100:00000001:3.0:1775775274.299681:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775274.299687:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12156 00000400:00000200:3.0:1775775274.299696:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775274.299703:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775274.299707:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775274.299711:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775274.299715:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775274.299720:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775274.299726:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775274.299730:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775274.299732:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775274.299735:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775274.299737:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775274.299741:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775274.299746:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775274.299754:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775274.299758:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775274.299762:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775274.299765:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775274.299768:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775274.299773:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775274.299776:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775274.299779:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775274.299783:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775274.299787:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775274.299789:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775274.299791:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775274.299795:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775274.299798:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775274.299819:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775775274.299822:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775274.299824:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775274.299825:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775274.299827:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775274.299829:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775274.299832:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775274.299837:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:3.0:1775775274.299844:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775274.299850:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710327296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775274.299864:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775274.299869:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775274.299875:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710327296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775274.299902:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775274.299906:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775274.299910:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775274.299913:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775274.299918:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775274.299925:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12167) 00000800:00000200:1.1:1775775274.300231:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775274.300233:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775775274.300495:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775274.300497:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00010000:00000001:3.0:1775775274.427225:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775274.427229:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775274.427233:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775274.427234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775274.427236:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775274.427237:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775275.451232:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775275.451238:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775275.451243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775275.451244:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775275.451246:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775275.451247:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775276.475225:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775276.475231:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775276.475236:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775276.475240:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775276.475242:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775276.475245:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775276.475247:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775276.475249:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775276.475251:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775276.475252:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775276.475255:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775276.475256:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775276.475256:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775276.475257:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775276.475258:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775276.475259:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775276.475260:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775276.475261:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775277.499230:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775277.499235:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775277.499239:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775277.499240:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775277.499244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775277.499245:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775278.523219:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775278.523224:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775278.523237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775278.523239:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775278.523241:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775278.523242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775278.523260:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775278.523263:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775278.523265:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775279.419230:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775279.419239:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775279.419242:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775279.419247:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775279.419251:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775279.419259:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12162 00000400:00000200:3.0:1775775279.419268:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775279.419275:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775279.419280:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775279.419283:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775279.419286:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775279.419288:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775279.419289:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775279.419292:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775279.419293:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775279.419296:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775279.419298:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775279.419299:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775279.419301:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775279.419304:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775279.419305:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775279.419308:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775279.419310:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775279.419312:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775279.419315:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775279.419316:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775279.419319:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775279.419321:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775279.419323:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775279.419325:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775279.419327:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775279.419335:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73b80. 00000020:00000040:3.0:1775775279.419337:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775279.419341:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775279.419342:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775279.419345:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775279.419347:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775279.419351:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775279.419359:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000001:3.0:1775775279.419364:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775279.419372:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73b80 x1862022710327424/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775279.419380:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775279.419383:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775279.419387:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c52008f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73b80 x1862022710327424/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775279.419404:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775279.419407:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775279.419409:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775279.419410:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775279.419412:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775279.419413:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775279.419416:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12162 00000400:00000200:3.0:1775775279.419418:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775279.419420:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775279.419422:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775279.419424:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775279.419425:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775279.419427:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775279.419428:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775279.419430:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775279.419430:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775279.419431:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775279.419433:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775279.419434:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775279.419436:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775279.419438:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775279.419440:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775279.419441:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775279.419443:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775279.419444:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775279.419446:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775279.419448:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775279.419451:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775279.419456:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775279.419458:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775279.419462:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775279.419466:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775279.419473:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775279.419478:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775279.419486:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775775279.419492:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775279.419499:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775279.419504:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775279.419508:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775279.419511:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775279.419515:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775279.419531:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000100:00000001:3.0:1775775279.419537:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775279.419544:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710327552/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775279.419560:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775279.419565:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775279.419582:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710327552/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775279.419602:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775279.419606:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775279.419614:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775279.419617:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775279.419622:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775279.419636:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775279.419642:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12162 00000400:00000200:3.0:1775775279.419652:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775279.419661:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775279.419664:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775279.419668:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775279.419670:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775279.419674:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775279.419679:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775279.419680:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775279.419683:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775279.419686:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775279.419688:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000800:00000200:1.1:1775775279.419691:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:3.0:1775775279.419692:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775279.419696:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:1.1:1775775279.419696:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000100:00000040:3.0:1775775279.419703:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775279.419708:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775279.419726:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775279.419729:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775279.419730:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775279.419733:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775279.419737:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775279.419741:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775279.419746:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775279.419749:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775279.419753:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775279.419757:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775279.419760:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775279.419763:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775279.419766:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775775279.419770:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775279.419775:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775279.419777:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775279.419781:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775279.419783:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775279.419785:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775279.419790:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:3.0:1775775279.419795:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775279.419802:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710327680/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775279.419826:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775279.419828:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775279.419833:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710327680/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775279.419843:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775279.419846:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775279.419851:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775279.419853:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775279.419856:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775279.419861:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12172) 00000800:00000200:1.1:1775775279.420087:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775279.420089:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775775279.420326:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775279.420327:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00010000:00000001:3.0:1775775279.547238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775279.547245:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775279.547250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775279.547251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775279.547255:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775279.547257:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775280.571202:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775280.571208:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775280.571213:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775280.571214:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775280.571217:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775280.571218:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775281.595201:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775281.595207:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775281.595209:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775281.595212:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775281.595217:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775281.595218:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775281.595220:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775281.595221:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775281.595224:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775281.595225:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775281.595228:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775281.595229:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775282.619268:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775282.619274:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775282.619279:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775282.619280:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775282.619283:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775282.619285:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775283.643237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775283.643242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775283.643246:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775283.643247:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775283.643250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775283.643251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775283.643268:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775283.643270:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775283.643272:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775775284.539739:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775284.539743:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775775284.540104:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775284.540106:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775775284.540306:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775284.540308:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00010000:00000001:3.0:1775775284.667251:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775284.667256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775284.667261:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775284.667263:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775284.667267:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775284.667268:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775285.691206:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775285.691212:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775285.691217:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775285.691219:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775285.691223:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775285.691224:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775286.715253:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775286.715258:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775286.715270:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775286.715273:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775286.715274:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775286.715277:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775286.715279:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775286.715281:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775286.715282:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775286.715284:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775286.715287:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775286.715288:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775286.715289:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775286.715290:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775286.715291:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775286.715292:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775286.715293:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775286.715295:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775287.739233:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775287.739238:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775287.739242:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775287.739243:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775287.739247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775287.739248:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775288.763236:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775288.763242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775288.763246:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775288.763246:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775288.763249:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775288.763250:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775288.763270:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775288.763271:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775288.763274:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775289.659276:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775289.659285:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775289.659288:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775289.659291:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775289.659295:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775289.659302:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12172 00000400:00000200:3.0:1775775289.659317:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775289.659327:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775289.659333:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775289.659338:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775289.659341:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775289.659344:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775289.659352:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775289.659357:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775289.659359:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775289.659362:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775289.659366:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775289.659375:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775289.659378:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775289.659381:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775289.659382:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775289.659383:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775289.659385:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775289.659387:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775289.659390:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775289.659392:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775289.659395:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775289.659398:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775289.659401:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775289.659402:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775289.659404:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775289.659413:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775775289.659416:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775289.659419:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775289.659420:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775289.659422:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775289.659425:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775289.659428:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775289.659434:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000100:00000001:3.0:1775775289.659440:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775289.659447:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710328192/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775289.659455:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775289.659458:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775289.659461:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710328192/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775289.659481:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775289.659483:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775289.659486:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775289.659487:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775289.659489:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775289.659491:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775289.659493:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12172 00000400:00000200:3.0:1775775289.659496:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775289.659498:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775289.659500:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775289.659503:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775289.659505:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775289.659506:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775289.659508:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775289.659511:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775289.659513:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775289.659515:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775289.659517:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775289.659519:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775289.659522:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775289.659526:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775289.659532:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775289.659549:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775289.659555:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775289.659561:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775289.659566:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775289.659571:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775289.659576:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775289.659583:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775289.659588:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775289.659592:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775289.659598:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775289.659606:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775289.659609:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775289.659612:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000020:00000040:3.0:1775775289.659617:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775289.659622:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775289.659625:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775289.659628:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775289.659630:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775289.659633:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775289.659637:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000100:00000001:3.0:1775775289.659642:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775289.659648:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710328320/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775289.659658:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775289.659662:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775289.659670:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710328320/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775289.659685:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775289.659689:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775289.659693:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775289.659696:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775289.659699:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775289.659704:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775289.659710:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12172 00000400:00000200:3.0:1775775289.659719:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775289.659727:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775289.659733:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775289.659741:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775289.659748:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775289.659754:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775289.659758:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775289.659764:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775289.659768:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775289.659771:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775289.659774:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775289.659777:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775289.659782:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775289.659785:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775289.659788:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775289.659791:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775289.659796:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775289.659798:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775289.659802:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775289.659805:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775289.659808:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:0.1:1775775289.659810:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:3.0:1775775289.659811:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775289.659816:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775289.659818:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:0.1:1775775289.659818:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000008:00100000:3.0:1775775289.659821:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775289.659824:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775289.659828:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775289.659831:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775775289.659835:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775289.659838:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775289.659840:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775289.659841:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775289.659843:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775289.659845:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775289.659849:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000100:00000001:3.0:1775775289.659854:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775289.659859:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710328448/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775289.659876:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775289.659879:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775289.659887:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710328448/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775289.659904:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775289.659909:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775289.659914:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775289.659918:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775289.659925:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775289.659932:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12182) 00000800:00000200:0.1:1775775289.660296:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775289.660298:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775775289.660481:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775289.660483:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00010000:00000001:3.0:1775775289.787225:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775289.787230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775289.787234:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775289.787235:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775289.787238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775289.787239:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775290.811231:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775290.811237:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775290.811244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775290.811245:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775290.811249:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775290.811250:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775291.835210:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775291.835219:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775291.835221:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775291.835225:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775291.835230:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775291.835233:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775291.835235:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775291.835238:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775291.835243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775291.835245:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775291.835249:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775291.835250:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775292.859227:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775292.859234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775292.859240:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775292.859241:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775292.859244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775292.859245:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775293.883200:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775293.883207:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775293.883213:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775293.883215:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775293.883227:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775293.883229:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775293.883252:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775293.883255:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775293.883258:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775294.779255:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775775294.779258:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1775775294.779265:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775294.779265:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775294.779269:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775294.779270:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1775775294.779276:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:2.0:1775775294.779277:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775294.779282:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775294.779283:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775775294.779286:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775294.779289:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775294.779291:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775775294.779293:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12177 00000100:00000001:3.0:1775775294.779294:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775775294.779305:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775294.779313:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775294.779317:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775294.779320:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775294.779324:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775294.779327:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775294.779329:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775294.779333:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775294.779335:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775294.779337:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775294.779340:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775294.779343:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775294.779345:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775294.779348:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775294.779350:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775294.779352:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775294.779354:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775294.779357:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775294.779360:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775294.779364:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775294.779367:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775294.779370:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775294.779373:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775294.779376:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775294.779378:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775294.779386:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775775294.779390:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775294.779394:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775294.779396:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775294.779398:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775294.779401:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775294.779405:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775294.779414:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775775294.779420:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775294.779430:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710328576/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775294.779440:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775294.779478:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775294.779485:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710328576/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775294.779507:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775294.779510:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775294.779513:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775294.779515:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775294.779516:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775294.779519:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775294.779521:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12177 00000400:00000200:2.0:1775775294.779525:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775294.779529:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775294.779541:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775294.779545:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775294.779547:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775294.779549:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775294.779552:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775294.779555:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775294.779556:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775294.779558:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775294.779560:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775294.779563:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775294.779566:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775294.779574:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775294.779577:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775294.779584:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775294.779588:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775294.779591:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775294.779601:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775294.779617:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775294.779624:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775294.779634:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775294.779639:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775294.779647:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775294.779653:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775294.779667:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775294.779679:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775294.779687:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775775294.779692:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775294.779696:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775294.779699:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775294.779705:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775294.779711:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775294.779714:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775294.779723:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775294.779728:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775294.779741:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710328704/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775294.779753:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775294.779759:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775294.779765:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710328704/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775294.779780:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775294.779785:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775294.779794:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775294.779799:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775294.779801:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775294.779807:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775294.779814:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12177 00000400:00000200:2.0:1775775294.779830:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775294.779846:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775294.779854:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775294.779866:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775294.779871:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775294.779877:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775294.779882:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775294.779889:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775294.779890:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775294.779893:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775294.779897:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775294.779905:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775294.779911:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775294.779921:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775294.779929:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775294.779932:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775294.779935:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775294.779941:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775294.779951:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775294.779957:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775294.779960:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775294.779967:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775294.779974:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775294.779982:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775294.779987:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775294.780006:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775294.780010:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775294.780014:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775775294.780019:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775294.780029:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775294.780032:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775294.780039:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775294.780042:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775294.780045:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775294.780053:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775775294.780059:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775294.780068:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710328832/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775294.780085:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775294.780089:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775294.780100:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710328832/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1775775294.780102:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775294.780111:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1775775294.780128:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775294.780167:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775294.780173:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775294.780176:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775294.780182:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775294.780191:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12187) 00000800:00000200:0.1:1775775294.780700:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775294.780703:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775775294.781003:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775294.781005:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00010000:00000001:3.0:1775775294.907237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775294.907242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775294.907247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775294.907249:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775294.907253:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775294.907254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775295.803195:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775295.803202:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775295.803204:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775295.803209:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775295.803213:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775295.803217:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775295.803221:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775295.803223:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775295.803226:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775295.931217:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775295.931221:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775295.931225:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775295.931226:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775295.931229:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775295.931230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775296.827212:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775296.827222:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775296.827224:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775296.827231:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775296.827235:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775296.827239:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775296.827242:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775296.827244:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775296.827246:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775296.955273:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775296.955281:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775296.955289:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775296.955294:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775296.955296:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775296.955301:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775296.955305:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775296.955308:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775296.955311:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775296.955314:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775296.955319:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775296.955321:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775296.955322:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775296.955324:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775296.955326:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775296.955327:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775296.955329:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775296.955331:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775297.851214:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775297.851223:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775297.851224:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775297.851231:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775297.851234:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775297.851238:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775297.851240:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775297.851243:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775297.851245:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775297.979274:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775297.979282:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775297.979295:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775297.979297:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775297.979303:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775297.979304:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775775298.683261:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775775298.683276:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775775298.683286:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000100:00000001:3.0:1775775298.875220:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775298.875227:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775298.875229:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775298.875235:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775298.875239:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775298.875243:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775298.875246:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775298.875248:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775298.875251:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775299.003220:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775299.003227:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775299.003231:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775299.003232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775299.003236:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775299.003237:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775299.003260:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775299.003262:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775299.003265:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775299.899222:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1775775299.899223:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775775299.899229:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1775775299.899230:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775299.899232:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775299.899233:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775299.899237:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775775299.899239:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775299.899243:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775775299.899244:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775299.899248:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775775299.899250:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12182 00000100:00000001:3.0:1775775299.899254:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775299.899258:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775775299.899260:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775775299.899263:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775775299.899274:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775299.899277:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775299.899281:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775299.899284:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775299.899286:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775299.899288:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775299.899291:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775299.899292:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775299.899294:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775299.899296:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775299.899297:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775299.899299:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775299.899301:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775299.899302:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775299.899304:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775299.899305:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775299.899307:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775299.899309:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775299.899311:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775299.899313:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775299.899315:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775299.899317:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775299.899319:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775299.899320:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775299.899328:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775775299.899330:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775299.899333:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775299.899334:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775299.899336:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775299.899338:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775299.899341:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775299.899347:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775299.899354:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775299.899373:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710328960/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775299.899381:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775299.899384:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775299.899388:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710328960/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775299.899405:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775299.899407:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775299.899409:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775299.899410:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775299.899412:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775299.899414:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775299.899415:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12182 00000400:00000200:2.0:1775775299.899418:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775299.899420:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775299.899422:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775299.899424:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775299.899425:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775299.899426:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775299.899428:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775299.899430:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775299.899430:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775299.899431:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775299.899433:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775299.899435:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775299.899436:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775299.899438:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775299.899440:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775299.899441:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775299.899442:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775299.899444:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775299.899446:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775299.899448:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775299.899449:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775299.899454:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775299.899457:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775299.899459:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775299.899462:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775299.899465:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775299.899466:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775299.899469:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775775299.899471:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775299.899472:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775299.899473:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775299.899475:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775299.899476:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775299.899477:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775299.899480:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775775299.899484:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775299.899491:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710329088/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775299.899504:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775299.899506:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775299.899518:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710329088/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775299.899568:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775299.899574:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775299.899580:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775299.899584:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775299.899588:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775299.899593:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775299.899601:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12182 00000400:00000200:2.0:1775775299.899611:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775299.899617:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775299.899624:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775299.899635:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775299.899639:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775299.899641:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775299.899642:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775299.899644:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775299.899645:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775299.899647:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775299.899650:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775299.899653:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775299.899658:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775299.899661:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775299.899663:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775299.899664:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775299.899667:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775299.899671:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775299.899676:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775299.899678:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775299.899681:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775299.899686:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775299.899689:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775299.899690:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775299.899691:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775299.899694:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775299.899703:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775299.899706:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775775299.899708:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775299.899710:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775299.899715:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775299.899727:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775299.899730:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775299.899735:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775299.899743:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775299.899752:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775299.899765:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710329216/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775299.899778:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775299.899783:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775299.899793:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710329216/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775299.899808:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775299.899814:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775299.899818:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775299.899820:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775299.899830:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775299.899842:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12192) 00000800:00000200:0.1:1775775299.899954:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775299.899958:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775775299.900520:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775299.900523:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775775299.900736:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775299.900738:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00010000:00000001:3.0:1775775300.027207:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775300.027213:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775300.027219:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775300.027220:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775300.027224:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775300.027225:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775300.923229:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775300.923235:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775300.923237:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775300.923241:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775300.923245:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775300.923247:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775300.923250:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775300.923251:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775300.923253:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775301.051206:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775301.051211:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775301.051215:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775301.051216:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775301.051220:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775301.051220:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775301.947210:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775301.947216:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775301.947218:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775301.947223:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775301.947227:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775301.947231:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775301.947234:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775301.947236:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775301.947239:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775302.075215:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775302.075223:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775302.075225:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775302.075229:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775302.075233:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775302.075237:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775302.075239:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775302.075242:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775302.075246:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775302.075248:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775302.075251:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775302.075253:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775302.971237:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775302.971242:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775302.971244:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775302.971248:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775302.971252:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775302.971255:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775302.971257:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775302.971259:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775302.971261:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775303.099205:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775303.099209:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775303.099213:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775303.099214:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775303.099217:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775303.099218:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775303.995236:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775303.995245:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775303.995248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775303.995254:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775303.995259:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775303.995263:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775303.995266:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775303.995269:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775303.995272:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775304.123267:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775304.123274:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775304.123301:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775304.123303:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775304.123307:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775304.123309:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775304.123335:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775304.123338:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775304.123341:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775305.019220:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775305.019228:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775305.019230:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775305.019235:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775305.019239:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775775305.019240:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775775305.019243:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775305.019251:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1775775305.019252:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775305.019255:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775305.019256:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775775305.019262:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775305.019263:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775305.019282:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775305.019289:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12187 00000400:00000200:2.0:1775775305.019298:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775305.019305:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775305.019307:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775305.019309:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775305.019312:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775305.019315:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775305.019316:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775305.019319:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775305.019321:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775305.019323:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775305.019325:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775305.019327:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775305.019328:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775305.019330:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775305.019331:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775305.019333:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775305.019334:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775305.019336:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775305.019339:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775305.019340:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775305.019342:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775305.019344:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775305.019346:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775305.019348:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775305.019349:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775305.019357:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775775305.019359:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775305.019362:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775305.019363:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775305.019365:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775305.019367:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775305.019370:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775305.019376:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775775305.019381:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775305.019387:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710329344/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775305.019395:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775305.019399:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775305.019402:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710329344/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775305.019420:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775305.019423:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775305.019425:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775305.019426:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775305.019427:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775305.019429:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775305.019431:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12187 00000400:00000200:2.0:1775775305.019433:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775305.019436:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775305.019438:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775305.019440:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775305.019442:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775305.019447:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775305.019449:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775305.019452:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775305.019453:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775305.019455:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775305.019456:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775305.019458:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775305.019459:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775305.019462:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775305.019468:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775305.019473:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775305.019480:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775305.019482:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775305.019491:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775305.019503:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775305.019513:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775305.019524:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775305.019528:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775305.019529:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775305.019533:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775305.019545:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775305.019553:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775305.019557:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775775305.019559:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775305.019565:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775305.019567:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775305.019569:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775305.019570:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775305.019573:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775305.019581:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775775305.019588:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775305.019596:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710329472/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775305.019603:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775305.019606:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775305.019618:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710329472/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775305.019625:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775305.019629:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775305.019638:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775305.019641:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775305.019643:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775305.019646:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775305.019649:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12187 00000400:00000200:2.0:1775775305.019658:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775305.019661:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775305.019663:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775305.019670:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775305.019672:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775305.019674:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775305.019675:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775305.019676:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775305.019679:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775305.019686:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775305.019688:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775305.019690:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775305.019691:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775305.019693:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775305.019695:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775305.019699:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775305.019704:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775305.019706:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775305.019711:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775305.019715:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775305.019717:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775305.019719:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775305.019723:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775305.019730:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775305.019734:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775305.019743:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775305.019752:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775305.019757:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775775305.019762:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775305.019766:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775305.019767:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775305.019775:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775305.019777:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775305.019783:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775305.019790:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775775305.019795:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775305.019799:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710329600/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775305.019818:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775305.019821:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775305.019831:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710329600/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775305.019853:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775305.019861:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775305.019872:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775305.019879:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775305.019891:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775305.019900:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12197) 00000800:00000200:0.1:1775775305.019938:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775305.019941:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775775305.020357:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775305.020360:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775775305.020561:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775305.020563:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00010000:00000001:3.0:1775775305.147212:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775305.147219:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775305.147225:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775305.147226:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775305.147229:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775305.147230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775306.043301:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775306.043310:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775306.043311:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775306.043317:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775306.043321:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775306.043325:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775306.043327:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775306.043330:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775306.043332:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775306.171331:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775306.171338:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775306.171344:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775306.171345:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775306.171348:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775306.171350:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775307.067329:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775307.067337:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775307.067339:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775307.067344:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775307.067349:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775307.067353:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775307.067357:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775307.067360:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775307.067363:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775307.195212:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775307.195219:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775307.195225:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775307.195239:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775307.195242:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775307.195245:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775307.195248:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775307.195250:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775307.195252:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775307.195254:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775307.195257:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775307.195258:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775307.195259:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775307.195260:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775307.195262:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775307.195263:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775307.195264:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775307.195265:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775308.091270:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775308.091278:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775308.091280:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775308.091286:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775308.091290:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775308.091294:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775308.091297:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775308.091299:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775308.091302:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775308.219254:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775308.219259:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775308.219263:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775308.219264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775308.219267:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775308.219268:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775309.115267:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775309.115275:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775309.115278:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775309.115284:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775309.115288:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775309.115292:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775309.115295:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775309.115298:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775309.115300:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775309.243258:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775309.243263:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775309.243267:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775309.243268:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775309.243271:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775309.243272:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775309.243292:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775309.243294:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775309.243296:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775310.139262:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775775310.139265:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775310.139270:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1775775310.139272:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775310.139272:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775310.139278:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1775775310.139278:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775310.139282:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775775310.139283:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775310.139285:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775310.139288:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775775310.139289:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775310.139292:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775310.139294:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775310.139297:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12192 00000400:00000200:2.0:1775775310.139307:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775310.139314:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775310.139318:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775310.139322:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775310.139325:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775310.139328:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775310.139331:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775310.139335:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775310.139337:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775310.139340:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775310.139342:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775310.139345:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775310.139348:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775310.139351:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775310.139353:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775310.139355:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775310.139357:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775310.139359:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775310.139362:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775310.139365:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775310.139368:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775310.139371:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775310.139374:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775310.139376:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775310.139378:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775310.139387:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775775310.139390:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775310.139394:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775310.139396:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775310.139398:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775310.139401:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775310.139405:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775310.139413:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775310.139419:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775310.139428:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710329728/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775310.139438:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775310.139443:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775310.139449:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710329728/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775310.139473:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775310.139475:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775310.139478:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775310.139479:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775310.139480:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775310.139482:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775310.139484:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12192 00000400:00000200:2.0:1775775310.139487:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775310.139489:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775310.139491:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775310.139493:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775310.139495:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775310.139496:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775310.139498:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775310.139500:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775310.139501:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775310.139502:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775310.139504:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775310.139507:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775310.139509:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775310.139511:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775310.139513:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775310.139515:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775310.139516:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775310.139518:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775310.139520:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775310.139522:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775310.139525:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775310.139534:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775310.139538:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775310.139542:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775310.139547:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775310.139554:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775310.139557:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775310.139566:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775775310.139575:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775310.139582:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775310.139587:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775310.139592:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775310.139595:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775310.139599:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775310.139613:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775775310.139619:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775310.139629:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710329856/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775310.139650:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775310.139655:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775310.139662:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710329856/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775310.139682:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775310.139686:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775310.139702:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775310.139705:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775310.139709:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775310.139713:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775310.139718:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12192 00000400:00000200:2.0:1775775310.139732:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775310.139748:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775310.139755:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775310.139768:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775310.139776:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775310.139782:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775310.139791:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775310.139796:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775310.139797:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775310.139799:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775310.139804:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775310.139810:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775310.139813:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775310.139819:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775310.139828:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775310.139830:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775310.139837:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775310.139844:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775310.139854:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775310.139866:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775310.139874:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775310.139883:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775310.139886:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775310.139889:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775310.139891:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775310.139901:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775310.139909:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775310.139917:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775775310.139924:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775310.139928:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775310.139929:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775310.139930:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775310.139931:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775310.139934:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775310.139944:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775775310.139950:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775310.139957:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710329984/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775310.139968:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775310.139974:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775310.139983:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710329984/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1775775310.139993:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775310.140004:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000100:00000001:2.0:1775775310.140008:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775310.140013:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775310.140024:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775310.140033:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775310.140040:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775310.140046:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12202) 00000800:00000200:0.1:1775775310.140526:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775310.140528:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775775310.140768:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775310.140770:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00010000:00000001:3.0:1775775310.267274:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775310.267280:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775310.267286:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775310.267287:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775310.267292:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775310.267293:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775311.163234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775311.163240:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775311.163241:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775311.163245:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775311.163248:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775311.163251:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775311.163254:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775311.163256:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775311.163257:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775311.291225:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775311.291232:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775311.291234:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775311.291238:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775311.291242:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775311.291245:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775311.291247:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775311.291250:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775311.291253:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775311.291255:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775311.291259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775311.291261:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775312.187251:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775312.187258:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775312.187261:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775312.187266:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775312.187269:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775312.187274:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775312.187277:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775312.187279:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775312.187282:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775312.315277:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775312.315283:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775312.315288:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775312.315289:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775312.315293:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775312.315294:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775313.211300:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775313.211306:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775313.211308:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775313.211314:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775313.211318:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775313.211321:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775313.211323:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775313.211325:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775313.211328:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775313.339218:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775313.339224:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775313.339229:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775313.339231:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775313.339235:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775313.339237:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775314.235210:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775314.235216:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775314.235218:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775314.235222:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775314.235225:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775314.235228:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775314.235230:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775314.235232:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775314.235233:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775314.363247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775314.363251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775314.363255:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775314.363256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775314.363259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775314.363260:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775314.363279:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775314.363281:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775314.363283:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775315.259260:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775315.259269:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775315.259273:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775315.259285:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775315.259301:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775315.259310:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775315.259321:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775315.259326:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775315.259334:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775315.259745:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775315.260001:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775315.260207:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00010000:00000001:3.0:1775775315.387284:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775315.387292:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775315.387297:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775315.387298:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775315.387301:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775315.387303:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775316.283229:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775316.283236:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775316.283237:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775316.283241:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775316.283244:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775316.283247:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775316.283250:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775316.283252:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775316.283254:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775316.411218:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775316.411223:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775316.411227:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775316.411229:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775316.411231:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775316.411234:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775316.411236:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775316.411237:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775316.411239:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775316.411240:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775316.411243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775316.411244:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775316.411245:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775316.411246:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775316.411247:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775316.411248:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775316.411249:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775316.411250:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775317.307327:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775317.307334:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775317.307336:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775317.307341:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775317.307345:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775317.307349:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775317.307351:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775317.307354:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775317.307357:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775317.435249:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775317.435254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775317.435259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775317.435260:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775317.435263:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775317.435264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775318.331280:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775318.331286:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775318.331288:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775318.331293:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775318.331296:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775318.331298:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775318.331300:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775318.331302:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775318.331304:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1775775318.331418:0:2400:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:2.0:1775775318.331455:0:2400:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1775775318.331466:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775318.331470:0:2400:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:2.0:1775775318.331475:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775318.331479:0:2400:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:2.0:1775775318.331485:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775318.331489:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:2.0:1775775318.331494:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775318.331500:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775318.331506:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3550 00000400:00000010:2.0:1775775318.331508:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3550. 00000100:00000001:2.0:1775775318.331515:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775318.331525:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f622e061f80 x1862022710330112/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775318.331570:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775318.331577:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061f80 x1862022710330112/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775340 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775318.331586:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775318.331588:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775775318.331598:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775318.331600:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230600. 00000400:00000200:2.0:1775775318.331603:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775318.331608:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775318.331611:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3440 00000400:00000010:2.0:1775775318.331613:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3440. 00000100:00000001:2.0:1775775318.331615:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775318.331620:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f622e063b80 x1862022710330240/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775318.331629:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775318.331632:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e063b80 x1862022710330240/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775340 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775318.331638:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775318.331640:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775775318.331644:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775318.331646:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230400. 00000400:00000200:2.0:1775775318.331649:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775318.331651:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775318.331654:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f32a8 00000400:00000010:2.0:1775775318.331656:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f32a8. 00000100:00000001:2.0:1775775318.331665:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775318.331670:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f622e061500 x1862022710330368/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775318.331677:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775318.331679:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061500 x1862022710330368/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775340 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775318.331686:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775318.331687:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:3.0:1775775318.459229:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775318.459234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775318.459238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775318.459239:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775318.459241:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775318.459242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775319.355233:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775319.355242:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775319.355245:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775319.355252:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775319.355257:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775319.355263:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775319.355266:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775319.355268:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775319.355271:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775319.483236:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775319.483243:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775319.483248:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775319.483249:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775319.483252:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775319.483253:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775319.483276:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775319.483278:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775319.483281:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775320.379250:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775320.379259:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775320.379262:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775320.379267:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775320.379271:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775320.379274:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775320.379277:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775320.379279:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775320.379283:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775320.379911:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775320.379917:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775775320.380970:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775320.380973:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775775320.381313:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775320.381314:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00010000:00000001:3.0:1775775320.507211:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775320.507217:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775320.507222:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775320.507223:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775320.507227:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775320.507228:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775321.403211:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775321.403219:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775321.403222:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775321.403228:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775321.403232:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775321.403237:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775321.403239:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775321.403242:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775321.403246:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775321.531298:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775321.531309:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775321.531311:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775321.531316:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775321.531321:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775321.531323:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775321.531325:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775321.531328:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775321.531333:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775321.531334:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775321.531338:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775321.531339:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775322.427251:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775322.427257:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775322.427269:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775322.427275:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775322.427278:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775322.427282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775322.427284:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775322.427286:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775322.427289:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775322.555199:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775322.555203:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775322.555207:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775322.555207:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775322.555210:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775322.555210:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775323.451193:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775323.451201:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775323.451203:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775323.451208:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775323.451212:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775323.451215:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775323.451217:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775323.451220:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775323.451223:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775323.579244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775323.579248:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775323.579252:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775323.579253:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775323.579255:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775323.579256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775324.475236:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775324.475241:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775324.475243:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775324.475247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775324.475249:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775324.475252:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775324.475254:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775324.475255:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775324.475257:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775324.603216:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775324.603221:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775324.603226:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775324.603227:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775324.603231:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775324.603232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775324.603252:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775324.603254:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775324.603257:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775325.499238:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775325.499245:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775325.499247:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775325.499253:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775325.499257:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775325.499261:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775325.499265:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775325.499268:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775325.499272:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775775325.499803:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775325.499806:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775775325.500088:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775325.500090:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775775325.500274:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775325.500276:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00010000:00000001:3.0:1775775325.627242:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775325.627246:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775325.627250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775325.627251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775325.627253:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775325.627254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775326.523235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775326.523240:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775326.523241:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775326.523245:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775326.523248:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775326.523250:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775326.523252:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775326.523254:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775326.523255:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775326.651295:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775326.651302:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775326.651307:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775326.651311:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775326.651314:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775326.651317:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775326.651321:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775326.651323:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775326.651325:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775326.651326:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775326.651329:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775326.651331:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775326.651331:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775326.651332:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775326.651333:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775326.651334:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775326.651335:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775326.651336:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775327.547252:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775327.547259:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775327.547260:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775327.547266:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775327.547269:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775327.547272:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775327.547275:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775327.547278:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775327.547281:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775327.675247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775327.675252:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775327.675255:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775327.675256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775327.675258:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775327.675259:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775328.571222:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775328.571228:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775328.571229:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775328.571235:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775328.571238:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775328.571243:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775328.571245:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775328.571247:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775328.571250:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775328.699227:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775328.699232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775328.699235:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775328.699236:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775328.699239:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775328.699239:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775329.595234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775329.595241:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775329.595243:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775329.595249:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775329.595253:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775329.595257:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775329.595259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775329.595261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775329.595264:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775329.723243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775329.723248:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775329.723253:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775329.723254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775329.723257:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775329.723257:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775329.723281:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775329.723284:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775329.723286:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775330.619236:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775330.619241:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775330.619244:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775330.619247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775330.619250:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775330.619252:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775330.619255:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775330.619258:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775330.619260:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775775330.619776:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775330.619780:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775775330.620190:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775330.620192:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775775330.620439:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775330.620441:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00010000:00000001:3.0:1775775330.747218:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775330.747223:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775330.747229:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775330.747231:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775330.747234:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775330.747236:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775331.643261:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775331.643269:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775331.643270:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775331.643276:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775331.643280:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775331.643284:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775331.643286:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775331.643289:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775331.643291:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775331.771271:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775331.771280:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775331.771285:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775331.771288:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775331.771293:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775331.771294:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775331.771296:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775331.771298:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775331.771301:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775331.771302:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775331.771305:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775331.771305:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775332.667244:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775332.667249:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775332.667251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775332.667258:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775332.667261:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775332.667266:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775332.667268:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775332.667270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775332.667273:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775332.795223:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775332.795229:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775332.795233:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775332.795234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775332.795237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775332.795239:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775332.795308:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775332.795312:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775332.795314:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775333.691218:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775333.691223:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775333.691225:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775333.691231:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775333.691234:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775333.691237:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775333.691238:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775333.691241:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775333.691242:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775333.819197:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775333.819201:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775333.819205:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775333.819205:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775333.819208:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775333.819209:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775334.843230:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775334.843235:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775334.843238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775334.843239:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775334.843241:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775334.843242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775334.843260:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775334.843262:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775334.843265:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775335.739228:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775335.739234:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775335.739238:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775335.739242:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775335.739249:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775335.739257:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12218 00000400:00000200:2.0:1775775335.739265:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775335.739272:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775335.739275:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775335.739278:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775335.739281:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775335.739283:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775335.739285:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775335.739288:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775335.739289:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775335.739292:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775335.739295:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775335.739297:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775335.739299:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775335.739302:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775335.739304:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775335.739306:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775335.739308:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775335.739310:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775335.739314:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775335.739317:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775335.739320:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775335.739323:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775335.739325:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775335.739328:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775335.739330:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775335.739338:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775775335.739341:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775335.739345:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775335.739347:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775335.739350:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775335.739352:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775335.739356:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775335.739368:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775775335.739374:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775335.739382:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710331648/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775335.739392:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775335.739396:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775335.739401:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710331648/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775335.739421:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775335.739424:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775335.739428:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775335.739429:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775335.739431:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775335.739433:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775335.739436:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12218 00000400:00000200:2.0:1775775335.739441:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775335.739445:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775335.739447:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775335.739451:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775335.739454:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775335.739458:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775335.739461:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775335.739464:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775335.739466:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775335.739469:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775335.739472:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775335.739477:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775335.739489:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775335.739497:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775335.739501:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775335.739506:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775335.739509:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775335.739513:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775335.739518:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775335.739521:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775335.739523:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775335.739529:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775335.739533:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775335.739536:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775335.739541:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775335.739546:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775335.739548:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775335.739552:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775775335.739554:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775335.739557:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775335.739560:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775335.739561:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775335.739563:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775335.739565:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775335.739571:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775335.739575:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775335.739583:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710331776/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775335.739600:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775335.739604:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775335.739609:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710331776/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775335.739620:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775335.739623:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775335.739627:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775335.739630:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775335.739632:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775335.739635:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775335.739639:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12218 00000400:00000200:2.0:1775775335.739645:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775335.739651:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775335.739653:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775335.739657:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775335.739661:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775335.739665:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775335.739668:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775335.739671:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775335.739672:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775335.739675:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775335.739678:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775335.739680:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775335.739682:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775335.739685:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775335.739686:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775335.739687:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775335.739689:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775335.739690:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775335.739694:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775335.739698:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775335.739701:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775335.739703:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775335.739706:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775335.739708:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:0.1:1775775335.739708:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000008:00100000:2.0:1775775335.739710:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775335.739713:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775775335.739713:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000100:00000001:2.0:1775775335.739716:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775335.739720:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775775335.739722:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775335.739726:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775335.739726:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775335.739727:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775335.739728:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775335.739730:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775335.739749:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775335.739754:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775335.739759:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710331904/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775335.739768:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775335.739770:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775335.739775:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710331904/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775335.739793:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775335.739796:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775335.739801:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775335.739805:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775335.739812:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775335.739817:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12228) 00000800:00000200:0.1:1775775335.740080:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775335.740083:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775775335.740365:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775335.740367:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00010000:00000001:3.0:1775775335.867228:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775335.867232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775335.867236:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775335.867237:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775335.867240:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775335.867240:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775775336.571275:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775775336.571284:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775775336.571288:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00010000:00000001:3.0:1775775336.891235:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775336.891240:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775336.891245:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775336.891249:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775336.891250:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775336.891253:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775336.891256:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775336.891258:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775336.891260:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775336.891262:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775336.891265:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775336.891267:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775336.891269:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775336.891270:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775336.891271:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775336.891272:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775336.891274:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775336.891276:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775337.915197:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775337.915201:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775337.915204:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775337.915205:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775337.915207:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775337.915208:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775337.915262:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775337.915265:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775337.915268:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775338.939271:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775338.939278:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775338.939285:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775338.939286:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775338.939290:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775338.939291:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775339.963259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775339.963267:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775339.963273:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775339.963275:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775339.963288:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775339.963289:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775339.963314:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775339.963317:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775339.963320:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775340.859281:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775340.859288:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775340.859291:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775340.859295:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775340.859298:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775340.859305:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12223 00000400:00000200:3.0:1775775340.859314:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775340.859320:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775340.859324:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775340.859326:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775340.859329:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775340.859330:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775340.859332:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775340.859335:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775340.859337:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775340.859339:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775340.859342:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775340.859345:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775340.859347:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775340.859350:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775340.859352:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775340.859354:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775340.859356:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775340.859359:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775340.859362:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775340.859365:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775340.859369:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775340.859373:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775340.859376:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775340.859378:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775340.859381:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775340.859393:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775775340.859397:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775340.859402:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775340.859404:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775340.859406:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775340.859409:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775340.859423:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775340.859435:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000100:00000001:3.0:1775775340.859444:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775340.859455:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710332032/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775340.859466:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775340.859470:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775340.859473:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710332032/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775340.859490:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775340.859493:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775340.859497:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775340.859498:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775340.859500:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775340.859502:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775340.859504:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12223 00000400:00000200:3.0:1775775340.859508:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775340.859511:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775340.859513:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775340.859516:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775340.859518:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775340.859519:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775340.859521:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775340.859523:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775340.859524:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775340.859526:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775340.859527:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775340.859530:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775340.859532:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775340.859535:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775340.859537:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775340.859538:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775340.859540:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775340.859543:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775340.859545:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775340.859547:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775340.859549:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775340.859555:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775340.859557:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775340.859558:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775340.859563:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775340.859566:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775340.859567:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775340.859571:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775775340.859573:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775340.859575:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775340.859576:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775340.859578:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775340.859579:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775340.859581:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775340.859584:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000100:00000001:3.0:1775775340.859587:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775340.859591:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710332160/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775340.859597:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775340.859599:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775340.859602:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710332160/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775340.859617:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775340.859620:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775340.859624:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775340.859625:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775340.859627:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775340.859630:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775340.859632:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12223 00000400:00000200:3.0:1775775340.859634:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775340.859637:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775340.859640:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775340.859652:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775340.859654:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775340.859656:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775340.859658:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775340.859660:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775340.859661:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775340.859662:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775340.859664:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775340.859666:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775340.859669:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775340.859671:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775340.859673:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775340.859675:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775340.859677:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775340.859679:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775340.859682:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775340.859684:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775340.859685:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775340.859687:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775340.859689:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775340.859691:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775340.859692:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775340.859695:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775340.859696:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775340.859700:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775775340.859702:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775340.859704:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775340.859705:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775340.859707:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775340.859708:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775340.859710:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775340.859713:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000100:00000001:3.0:1775775340.859715:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775340.859719:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710332288/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775340.859725:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:0.1:1775775340.859726:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775775340.859727:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775340.859730:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710332288/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1775775340.859730:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000100:00000001:3.0:1775775340.859737:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775340.859740:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775340.859743:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775340.859746:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775340.859748:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775340.859753:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12233) 00000800:00000200:0.1:1775775340.860129:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775340.860132:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775775340.860455:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775340.860457:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00010000:00000001:3.0:1775775340.987267:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775340.987275:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775340.987282:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775340.987284:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775340.987287:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775340.987289:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775342.011237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775342.011243:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775342.011244:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775342.011247:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775342.011251:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775342.011253:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775342.011255:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775342.011257:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775342.011260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775342.011262:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775342.011265:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775342.011267:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775343.035271:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775343.035277:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775343.035284:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775343.035285:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775343.035289:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775343.035291:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775343.035360:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775343.035365:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775343.035369:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775344.059239:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775344.059244:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775344.059248:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775344.059249:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775344.059252:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775344.059253:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775345.083252:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775345.083258:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775345.083264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775345.083265:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775345.083269:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775345.083270:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775345.083295:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775345.083298:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775345.083301:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775345.979266:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775345.979276:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775345.979287:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775345.979291:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775345.979302:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775345.979312:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12228 00000400:00000200:3.0:1775775345.979323:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775345.979331:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775345.979335:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775345.979340:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775345.979344:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775345.979348:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775345.979350:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775345.979355:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775345.979357:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775345.979360:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775345.979362:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775345.979365:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775345.979367:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775345.979370:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775345.979373:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775345.979375:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775345.979377:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775345.979380:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775345.979384:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775345.979387:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775345.979390:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775345.979393:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775345.979396:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775345.979398:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775345.979400:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775345.979409:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775775345.979413:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775345.979417:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775345.979419:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775345.979421:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775345.979424:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775345.979428:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775345.979437:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000100:00000001:3.0:1775775345.979444:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775345.979454:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710332416/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775345.979464:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775345.979469:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775345.979474:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710332416/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775345.979498:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775345.979501:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775345.979505:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775345.979506:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775345.979508:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775345.979511:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775345.979514:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12228 00000400:00000200:3.0:1775775345.979519:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775345.979523:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775345.979526:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775345.979531:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775345.979536:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775345.979542:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775345.979547:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775345.979567:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775345.979569:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775345.979571:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775345.979574:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775345.979580:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775345.979586:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775345.979594:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775345.979600:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775345.979605:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775345.979611:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775345.979615:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775345.979619:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775345.979624:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775345.979629:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775345.979635:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775345.979640:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775345.979644:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775345.979649:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775345.979655:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775345.979659:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775345.979671:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000020:00000040:3.0:1775775345.979677:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775345.979684:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775345.979687:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775345.979691:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775345.979695:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775345.979698:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775345.979705:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:3.0:1775775345.979713:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775345.979722:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710332544/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775345.979742:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775345.979749:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775345.979759:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710332544/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775345.979784:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775345.979790:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775345.979805:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775345.979809:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:0.1:1775775345.979809:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00080000:3.0:1775775345.979813:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:0.1:1775775345.979814:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000100:00000001:3.0:1775775345.979822:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775345.979829:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12228 00000400:00000200:3.0:1775775345.979837:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775345.979847:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775345.979853:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775345.979861:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775345.979867:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775345.979873:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775345.979878:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775345.979884:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775345.979887:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775345.979891:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775345.979893:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775345.979897:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775345.979900:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775345.979905:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775345.979910:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775345.979913:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775345.979918:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775345.979920:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775345.979925:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775345.979927:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775345.979928:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775345.979931:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775345.979934:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775345.979935:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775345.979939:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775345.979943:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775345.979946:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775345.979952:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775775345.979957:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775345.979962:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775345.979965:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775345.979969:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775345.979971:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775345.979974:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775345.979980:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000001:3.0:1775775345.979986:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775345.979994:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710332672/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775345.980014:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775345.980018:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775345.980025:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710332672/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775345.980042:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775345.980048:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775345.980054:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775345.980060:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775345.980065:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775345.980071:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12238) 00000800:00000200:0.1:1775775345.980264:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775345.980266:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775775345.980478:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775345.980480:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00010000:00000001:3.0:1775775346.107214:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775346.107218:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775346.107222:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775346.107223:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775346.107226:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775346.107227:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775347.131234:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775347.131239:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775347.131243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775347.131246:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775347.131248:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775347.131251:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775347.131254:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775347.131255:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775347.131257:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775347.131258:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775347.131261:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775347.131262:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775347.131263:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775347.131264:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775347.131264:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775347.131265:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775347.131266:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775347.131268:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775348.155255:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775348.155260:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775348.155265:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775348.155265:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775348.155268:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775348.155269:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775348.155325:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775348.155329:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775348.155333:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775349.179265:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775349.179271:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775349.179277:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775349.179279:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775349.179283:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775349.179284:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775350.203250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775350.203255:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775350.203259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775350.203260:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775350.203263:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775350.203265:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775350.203289:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775350.203291:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775350.203293:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775775351.099928:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775351.099944:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775775351.100358:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775351.100360:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775775351.100633:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775351.100634:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00010000:00000001:3.0:1775775351.227236:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775351.227241:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775351.227245:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775351.227246:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775351.227250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775351.227251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775352.251231:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775352.251239:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775352.251241:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775352.251245:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775352.251249:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775352.251252:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775352.251254:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775352.251257:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775352.251260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775352.251262:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775352.251266:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775352.251267:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775353.275249:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775353.275256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775353.275262:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775353.275263:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775353.275268:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775353.275269:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775353.275334:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775353.275338:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775353.275340:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775354.299232:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775354.299238:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775354.299244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775354.299246:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775354.299250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775354.299252:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775355.323228:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775355.323233:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775355.323237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775355.323238:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775355.323242:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775355.323243:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775355.323262:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775355.323265:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775355.323267:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775356.219268:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775356.219276:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775356.219281:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775356.219286:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775356.219290:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775356.219298:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12238 00000400:00000200:2.0:1775775356.219307:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775356.219315:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775356.219318:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775356.219321:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775356.219324:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775356.219326:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775356.219328:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775356.219332:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775356.219334:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775356.219337:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775356.219339:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775356.219350:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775356.219353:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775356.219356:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775356.219358:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775356.219359:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775356.219361:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775356.219364:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775356.219366:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775356.219369:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775356.219372:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775356.219390:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775356.219393:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775356.219395:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775356.219397:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775356.219406:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775775356.219409:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775356.219413:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775356.219414:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775356.219417:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775356.219420:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775356.219424:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775356.219431:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775775356.219438:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775356.219447:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710333184/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775356.219455:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775356.219458:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775356.219462:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710333184/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775356.219482:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775356.219484:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775356.219486:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775356.219487:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775356.219489:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775356.219491:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775356.219492:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12238 00000400:00000200:2.0:1775775356.219495:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775356.219498:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775356.219500:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775356.219502:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775356.219504:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775356.219506:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775356.219508:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775356.219510:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775356.219510:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775356.219512:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775356.219513:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775356.219515:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775356.219516:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775356.219518:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775356.219522:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775356.219523:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775356.219527:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775356.219530:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775356.219534:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775356.219537:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775356.219540:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775356.219546:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775356.219550:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775356.219554:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775356.219559:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775356.219563:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775356.219567:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775356.219570:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775775356.219573:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775356.219577:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775356.219579:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775356.219584:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775356.219586:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775356.219589:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775356.219594:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775775356.219599:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775356.219605:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710333312/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775356.219619:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775356.219621:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775356.219624:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710333312/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775356.219636:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775356.219640:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775356.219645:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775356.219647:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775356.219648:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775356.219653:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775356.219655:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12238 00000400:00000200:2.0:1775775356.219658:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775356.219661:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775356.219664:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775356.219666:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775356.219668:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775356.219672:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775356.219674:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775356.219678:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775356.219681:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775356.219682:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775356.219683:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775356.219686:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775356.219688:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775356.219696:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775356.219700:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775356.219704:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775356.219709:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775356.219711:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775356.219718:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775356.219721:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775356.219723:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775356.219727:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775356.219729:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775356.219733:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775356.219735:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775356.219737:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775356.219741:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775356.219746:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775775356.219748:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775356.219750:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775356.219751:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775356.219754:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775356.219756:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775356.219758:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775356.219763:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775775356.219768:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775356.219774:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710333440/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775356.219785:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775356.219788:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775356.219795:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710333440/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775356.219800:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775356.219802:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775356.219805:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775356.219809:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775356.219813:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775356.219817:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12248) 00000800:00000200:0.1:1775775356.219868:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775356.219872:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1775775356.220296:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775356.220299:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1775775356.220618:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775356.220620:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 23 secs later) 00010000:00000001:3.0:1775775356.347191:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775356.347196:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775356.347199:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775356.347203:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775356.347205:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775356.347207:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775356.347209:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775356.347211:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775356.347213:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775356.347214:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775356.347217:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775356.347218:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775356.347219:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775356.347220:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775356.347221:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775356.347222:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775356.347223:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775356.347224:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775357.371242:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775357.371246:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775357.371250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775357.371251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775357.371254:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775357.371255:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775358.395212:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775358.395219:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775358.395225:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775358.395226:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775358.395231:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775358.395232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775358.395286:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775358.395288:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775358.395291:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775359.419221:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775359.419227:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775359.419231:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775359.419232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775359.419234:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775359.419236:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775360.443258:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775360.443263:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775360.443267:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775360.443268:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775360.443271:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775360.443273:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775360.443293:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775360.443295:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775360.443297:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775361.339234:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775361.339242:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775361.339246:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775361.339250:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775361.339253:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775361.339262:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12243 00000400:00000200:2.0:1775775361.339273:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775361.339280:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775361.339283:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775361.339298:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775361.339302:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775361.339304:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775361.339308:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775361.339312:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775361.339314:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775361.339317:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775361.339320:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775361.339323:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775361.339325:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775361.339329:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775361.339331:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775361.339340:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775361.339343:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775361.339346:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775361.339354:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775361.339357:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775361.339361:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775361.339364:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775361.339367:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775361.339370:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775361.339372:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775361.339381:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775775361.339385:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775361.339389:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775361.339391:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775361.339394:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775361.339396:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775361.339401:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775361.339409:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775361.339415:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775361.339423:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710333568/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775361.339432:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775361.339435:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775361.339439:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710333568/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775361.339456:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775361.339459:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775361.339462:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775361.339463:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775361.339464:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775361.339466:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775361.339468:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12243 00000400:00000200:2.0:1775775361.339472:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775361.339475:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775361.339477:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775361.339479:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775361.339481:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775361.339483:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775361.339486:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775361.339490:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775361.339492:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775361.339495:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775361.339497:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775361.339500:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775361.339502:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775361.339507:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775361.339513:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775361.339518:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775361.339523:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775361.339526:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775361.339529:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775361.339532:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775361.339537:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775361.339545:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775361.339549:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775361.339552:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775361.339557:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775361.339564:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775361.339570:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775361.339577:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775775361.339584:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775361.339589:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775361.339593:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775361.339599:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775361.339601:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775361.339604:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775361.339610:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775361.339614:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775361.339620:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710333696/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775361.339633:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775361.339636:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775361.339642:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710333696/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775361.339664:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775361.339668:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775361.339673:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775361.339676:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775361.339678:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775361.339684:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775361.339688:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12243 00000400:00000200:2.0:1775775361.339695:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775361.339699:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775361.339702:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775361.339704:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775361.339706:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775361.339710:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775361.339715:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775361.339717:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775361.339719:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775361.339722:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775361.339724:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775361.339727:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775361.339729:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775361.339734:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775361.339737:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775361.339742:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775361.339745:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775361.339750:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775361.339757:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775361.339763:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775361.339769:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775361.339774:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775361.339778:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775361.339782:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775361.339787:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775361.339793:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775361.339796:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775361.339803:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775775361.339807:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775361.339811:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775361.339814:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775361.339816:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775361.339818:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775361.339823:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775361.339831:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775775361.339839:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775775361.339842:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00100000:2.0:1775775361.339848:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710333824/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.1:1775775361.339849:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1775775361.339861:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775361.339866:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775361.339871:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710333824/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775361.339885:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775361.339887:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775361.339893:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775361.339897:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775361.339901:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775361.339906:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12254) 00000800:00000200:0.1:1775775361.340184:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775361.340186:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775775361.340438:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775361.340440:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00010000:00000001:3.0:1775775361.467220:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775361.467227:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775361.467230:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775361.467233:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775361.467238:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775361.467241:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775361.467243:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775361.467246:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775361.467249:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775361.467250:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775361.467258:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775361.467259:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775362.491388:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775362.491394:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775362.491398:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775362.491399:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775362.491402:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775362.491403:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775363.515224:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775363.515230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775363.515235:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775363.515236:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775363.515240:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775363.515241:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775363.515322:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775363.515325:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775363.515327:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775364.539284:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775364.539292:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775364.539298:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775364.539299:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775364.539303:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775364.539304:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775365.563236:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775365.563241:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775365.563245:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775365.563246:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775365.563249:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775365.563250:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775365.563269:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775365.563271:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775365.563273:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775366.459247:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775366.459258:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775366.459264:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775366.459270:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775366.459285:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775366.459295:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12249 00000400:00000200:2.0:1775775366.459308:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775366.459315:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775366.459318:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775366.459322:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775366.459325:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775366.459328:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775366.459330:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775366.459334:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775366.459336:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775366.459339:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775366.459342:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775366.459344:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775366.459345:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775366.459347:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775366.459348:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775366.459350:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775366.459351:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775366.459353:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775366.459356:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775366.459358:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775366.459361:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775366.459365:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775366.459367:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775366.459370:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775366.459372:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775366.459382:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775775366.459386:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775366.459390:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775366.459391:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775366.459394:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775366.459397:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775366.459401:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775366.459409:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775366.459415:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775366.459424:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710333952/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775366.459433:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775366.459438:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775366.459442:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710333952/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775366.459464:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775366.459466:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775366.459470:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775366.459471:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775366.459472:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775366.459474:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775366.459476:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12249 00000400:00000200:2.0:1775775366.459478:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775366.459480:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775366.459481:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775366.459483:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775366.459485:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775366.459486:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775366.459488:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775366.459489:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775366.459490:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775366.459491:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775366.459493:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775366.459495:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775366.459496:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775366.459498:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775366.459500:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775366.459503:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775366.459505:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775366.459507:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775366.459509:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775366.459511:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775366.459513:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775366.459519:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775366.459523:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775366.459526:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775366.459532:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775366.459538:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775366.459545:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775366.459553:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775775366.459561:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775366.459570:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775366.459574:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775366.459579:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775366.459583:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775366.459585:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775366.459591:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775775366.459603:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775366.459610:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710334080/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775366.459627:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775366.459629:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775366.459638:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710334080/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775366.459649:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775366.459653:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775366.459658:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775366.459663:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775366.459665:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775366.459667:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775366.459675:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12249 00000400:00000200:2.0:1775775366.459685:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775366.459686:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775366.459688:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775366.459697:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775366.459700:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775366.459703:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775366.459710:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775366.459714:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775366.459715:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775366.459720:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775366.459724:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775366.459729:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775366.459734:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775366.459744:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775366.459754:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775366.459765:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775366.459773:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775366.459777:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775366.459785:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775366.459792:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775366.459795:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775366.459800:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775366.459804:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775366.459809:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775366.459811:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775366.459815:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775366.459820:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775366.459828:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775775366.459830:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775366.459834:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775366.459838:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775366.459848:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775366.459851:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775366.459853:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775366.459858:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775775366.459863:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775366.459870:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710334208/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775366.459905:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775366.459912:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775366.459926:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710334208/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775366.459950:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775366.459961:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775366.459975:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775366.459997:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775366.460012:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:0.1:1775775366.460012:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775366.460022:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000100:00000040:2.0:1775775366.460027:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12259) 00000800:00000200:0.1:1775775366.460393:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775366.460396:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775775366.460607:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775366.460609:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00010000:00000001:3.0:1775775366.587274:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775366.587279:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775366.587283:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775366.587286:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775366.587288:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775366.587290:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775366.587292:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775366.587294:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775366.587296:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775366.587297:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775366.587301:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775366.587302:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775366.587302:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775366.587303:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775366.587304:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775366.587305:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775366.587306:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775366.587308:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775367.611255:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775367.611260:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775367.611265:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775367.611266:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775367.611268:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775367.611269:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775368.635290:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775368.635298:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775368.635305:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775368.635306:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775368.635309:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775368.635310:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775368.635374:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775368.635379:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775368.635383:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775369.659305:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775369.659315:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775369.659324:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775369.659326:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775369.659329:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775369.659330:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775370.683294:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775370.683300:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775370.683306:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775370.683307:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775370.683310:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775370.683311:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775370.683342:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775370.683345:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775370.683349:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775371.579444:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775371.579451:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775371.579454:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775371.579458:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775371.579463:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775371.579471:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12254 00000400:00000200:3.0:1775775371.579483:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775371.579491:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775371.579495:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775371.579499:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775371.579502:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775371.579504:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775371.579507:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775371.579511:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775371.579513:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775371.579517:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775371.579520:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775371.579523:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775371.579525:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775371.579534:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775371.579536:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775371.579539:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775371.579541:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775371.579544:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775371.579548:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775371.579551:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775371.579555:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775371.579559:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775371.579563:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775371.579566:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775371.579569:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775371.579581:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73100. 00000020:00000040:3.0:1775775371.579585:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775371.579590:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775371.579592:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775371.579596:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775371.579599:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775371.579604:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775371.579615:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000001:3.0:1775775371.579623:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775371.579635:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73100 x1862022710334336/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775371.579647:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775371.579653:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775371.579658:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f67e6552] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73100 x1862022710334336/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775371.579683:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775371.579687:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775371.579690:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775371.579692:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775371.579694:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775371.579696:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775371.579700:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12254 00000400:00000200:3.0:1775775371.579704:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775371.579708:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775371.579711:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775371.579715:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775371.579717:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775371.579720:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775371.579723:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775371.579730:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775371.579732:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775371.579734:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775371.579737:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775371.579740:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775371.579744:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775371.579749:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775371.579752:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775371.579756:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775371.579761:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775371.579766:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775371.579773:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775371.579778:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775371.579781:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775371.579789:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775371.579793:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775371.579796:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775371.579803:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775371.579808:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775371.579811:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775371.579819:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775775371.579823:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775371.579827:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775371.579829:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775371.579832:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775371.579834:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775371.579838:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775371.579845:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:3.0:1775775371.579849:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775371.579858:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710334464/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775371.579872:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775371.579876:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775371.579880:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710334464/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775371.579893:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775371.579899:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775371.579905:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775371.579907:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775371.579908:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775371.579911:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775371.579915:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12254 00000400:00000200:3.0:1775775371.579920:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775371.579925:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775371.579929:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775371.579936:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775371.579942:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775371.579947:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775371.579951:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775371.579956:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775371.579959:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775371.579963:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775371.579966:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775371.579969:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775371.579975:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775371.579981:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775371.579984:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775371.579986:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775371.579988:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775371.579992:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775371.579997:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775371.580000:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775371.580002:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775371.580006:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775371.580010:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775371.580012:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775371.580015:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775371.580019:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775371.580023:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775371.580031:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000020:00000040:3.0:1775775371.580036:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775371.580042:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775371.580045:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775371.580048:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775371.580051:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775371.580054:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775371.580061:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775775371.580066:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775371.580077:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710334592/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775371.580093:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775371.580097:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775371.580103:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710334592/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775371.580164:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775371.580168:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775371.580175:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775371.580180:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775371.580187:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:0.1:1775775371.580191:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775775371.580195:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12264) 00000800:00000200:0.1:1775775371.580197:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775775371.580591:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775371.580593:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775775371.580841:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775371.580844:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00010000:00000001:3.0:1775775371.707251:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775371.707259:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775371.707262:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775371.707265:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775371.707270:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775371.707273:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775371.707275:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775371.707277:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775371.707280:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775371.707281:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775371.707284:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775371.707285:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775372.731260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775372.731267:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775372.731272:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775372.731273:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775372.731277:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775372.731278:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775373.755257:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775373.755264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775373.755269:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775373.755270:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775373.755274:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775373.755275:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775373.755353:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775373.755358:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775373.755362:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775374.779291:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775374.779297:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775374.779303:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775374.779305:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775374.779309:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775374.779311:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775775375.483272:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775775375.483280:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775775375.483287:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00010000:00000001:3.0:1775775375.803270:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775375.803276:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775375.803295:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775375.803297:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775375.803301:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775375.803303:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775375.803329:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775375.803332:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775375.803335:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775376.699251:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775376.699259:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775376.699262:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775376.699267:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775376.699272:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775376.699280:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12259 00000400:00000200:2.0:1775775376.699288:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775376.699294:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775376.699297:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775376.699301:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775376.699303:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775376.699305:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775376.699307:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775376.699309:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775376.699311:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775376.699313:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775376.699315:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775376.699317:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775376.699318:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775376.699321:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775376.699322:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775376.699323:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775376.699325:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775376.699327:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775376.699329:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775376.699331:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775376.699333:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775376.699335:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775376.699337:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775376.699338:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775376.699340:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775376.699347:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775775376.699350:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775376.699352:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775376.699353:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775376.699356:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775376.699357:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775376.699361:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775376.699367:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775775376.699372:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775376.699379:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710334720/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775376.699387:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775376.699390:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775376.699393:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710334720/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775376.699410:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775376.699412:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775376.699414:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775376.699416:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775376.699417:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775376.699419:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775376.699420:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12259 00000400:00000200:2.0:1775775376.699422:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775376.699424:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775376.699426:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775376.699428:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775376.699429:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775376.699430:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775376.699432:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775376.699433:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775376.699434:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775376.699435:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775376.699437:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775376.699438:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775376.699440:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775376.699442:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775376.699444:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775376.699446:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775376.699447:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775376.699449:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775376.699451:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775376.699452:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775376.699453:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775376.699458:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775376.699461:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775376.699463:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775376.699467:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775376.699470:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775376.699471:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775376.699475:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775775376.699478:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775376.699482:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775376.699484:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775376.699489:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775376.699491:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775376.699494:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775376.699497:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775775376.699501:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775376.699507:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710334848/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775376.699520:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775376.699522:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775376.699528:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710334848/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775376.699539:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775376.699541:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775376.699545:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775376.699548:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775376.699550:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775376.699559:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775376.699564:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12259 00000400:00000200:2.0:1775775376.699575:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775376.699578:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775376.699581:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775376.699583:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775376.699589:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775376.699591:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775376.699592:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775376.699594:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775376.699596:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775376.699598:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775376.699600:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775376.699601:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775376.699602:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775376.699604:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775376.699605:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775376.699606:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775376.699607:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775376.699609:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775376.699610:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775376.699615:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775376.699620:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775376.699622:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775376.699625:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775376.699627:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775376.699628:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775376.699631:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775376.699633:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775376.699638:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775775376.699643:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775376.699647:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775376.699650:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775376.699651:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775376.699653:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775376.699656:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775376.699660:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775376.699663:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775376.699672:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710334976/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775376.699681:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775376.699684:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775376.699689:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710334976/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775376.699703:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775376.699707:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775376.699713:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775376.699719:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775376.699726:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775376.699734:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12269) 00000800:00000200:0.1:1775775376.699784:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775376.699787:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775775376.700261:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775376.700264:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775775376.700561:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775376.700564:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00010000:00000001:3.0:1775775376.827241:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775376.827247:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775376.827254:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775376.827258:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775376.827260:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775376.827264:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775376.827267:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775376.827270:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775376.827272:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775376.827274:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775376.827278:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775376.827280:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775376.827281:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775376.827283:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775376.827284:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775376.827286:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775376.827288:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775376.827290:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775377.851218:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775377.851225:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775377.851230:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775377.851232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775377.851235:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775377.851236:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775378.875267:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775378.875274:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775378.875280:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775378.875281:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775378.875285:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775378.875286:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775378.875349:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775378.875352:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775378.875355:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775379.899255:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775379.899262:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775379.899267:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775379.899268:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775379.899272:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775379.899273:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775380.923210:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775380.923217:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775380.923222:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775380.923223:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775380.923226:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775380.923226:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775380.923250:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775380.923253:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775380.923257:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775381.819253:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775381.819260:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775381.819264:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775381.819271:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775381.819275:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775381.819281:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775381.819285:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775381.819288:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775381.819292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775775381.819897:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775381.820197:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775381.820367:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00010000:00000001:3.0:1775775381.947281:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775381.947290:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775381.947292:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775381.947296:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775381.947301:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775381.947303:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775381.947305:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775381.947307:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775381.947310:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775381.947312:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775381.947315:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775381.947316:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775382.843308:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775382.843314:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775382.843315:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775382.843320:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775382.843322:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775382.843326:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775382.843328:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775382.843330:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775382.843332:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775382.971224:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775382.971230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775382.971236:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775382.971237:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775382.971242:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775382.971243:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775383.867274:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775383.867281:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775383.867284:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775383.867290:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775383.867294:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775383.867297:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775383.867300:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775383.867303:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775383.867305:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775383.995255:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775383.995261:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775383.995266:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775383.995267:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775383.995271:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775383.995272:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775383.995323:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775383.995326:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775383.995329:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775384.891247:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775384.891253:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775384.891255:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775384.891259:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775384.891261:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775384.891264:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775384.891266:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775384.891267:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775384.891269:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:3.0:1775775384.891423:0:2399:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:3.0:1775775384.891465:0:2399:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1775775384.891475:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775384.891479:0:2399:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:3.0:1775775384.891484:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775384.891488:0:2399:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:3.0:1775775384.891493:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775775384.891497:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c700. 00000400:00000200:3.0:1775775384.891502:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775775384.891507:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775775384.891513:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc07f8 00000400:00000010:3.0:1775775384.891515:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc07f8. 00000100:00000001:3.0:1775775384.891520:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775775384.891529:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6234918380 x1862022710335104/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775406 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775384.891553:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775775384.891558:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918380 x1862022710335104/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775406 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775384.891565:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775384.891567:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775775384.891576:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775775384.891579:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111ca00. 00000400:00000200:3.0:1775775384.891583:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775775384.891587:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775775384.891591:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:3.0:1775775384.891593:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:3.0:1775775384.891596:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775775384.891601:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f623491aa00 x1862022710335232/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775406 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775384.891610:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775775384.891615:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491aa00 x1862022710335232/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775406 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775384.891626:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775384.891628:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775775384.891636:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775775384.891643:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cf00. 00000400:00000200:3.0:1775775384.891648:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775775384.891653:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775775384.891673:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0cc0 00000400:00000010:3.0:1775775384.891678:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0cc0. 00000100:00000001:3.0:1775775384.891684:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775775384.891692:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6234918e00 x1862022710335360/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775406 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775384.891704:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775775384.891706:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918e00 x1862022710335360/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775406 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775384.891715:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775384.891716:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:3.0:1775775385.019238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775385.019242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775385.019246:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775385.019247:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775385.019250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775385.019251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775385.915225:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775385.915230:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775385.915232:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775385.915236:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775385.915239:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775385.915242:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775385.915244:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775385.915246:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775385.915249:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775386.043224:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775386.043229:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775386.043234:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775386.043235:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775386.043237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775386.043238:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775386.043258:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775386.043260:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775386.043263:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775386.939263:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775386.939273:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775386.939276:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775386.939284:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775386.939288:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775386.939293:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775386.939297:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775386.939301:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775386.939304:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775386.939328:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775386.939337:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775386.939340:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775386.939345:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775386.939349:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775386.939359:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12269 00000400:00000200:2.0:1775775386.939377:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775386.939385:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775386.939390:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775386.939394:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775386.939398:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775386.939401:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775386.939404:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775386.939408:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775386.939411:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775386.939414:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775386.939418:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775386.939421:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775386.939424:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775386.939427:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775386.939430:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775386.939432:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775386.939434:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775386.939438:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775386.939442:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775386.939445:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775386.939450:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775386.939454:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775386.939458:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775386.939461:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775386.939463:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775386.939472:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775775386.939476:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775386.939482:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775386.939484:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775386.939487:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775386.939491:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775386.939497:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775386.939507:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775775386.939516:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775386.939528:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710335488/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775386.939542:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775386.939549:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775386.939555:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710335488/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775386.939585:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775386.939588:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775386.939592:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775386.939594:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775386.939595:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775386.939599:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775386.939602:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12269 00000400:00000200:2.0:1775775386.939606:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775386.939610:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775386.939612:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775386.939616:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775386.939634:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775386.939638:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775386.939641:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775386.939645:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775386.939646:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775386.939648:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775386.939651:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775386.939653:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775386.939656:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775386.939660:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775386.939662:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775386.939665:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775386.939667:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775386.939672:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775386.939678:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775386.939685:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775386.939689:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775386.939698:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775386.939704:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775386.939707:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775386.939714:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775386.939720:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775386.939725:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775386.939734:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775775386.939739:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775386.939743:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775386.939746:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775386.939750:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775386.939752:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775386.939756:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775386.939762:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775775386.939767:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775386.939777:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710335616/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775386.939792:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775386.939797:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775386.939818:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710335616/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775386.939836:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775386.939852:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775386.939857:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775386.939861:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775386.939864:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775386.939871:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775386.939878:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12269 00000400:00000200:2.0:1775775386.939888:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775386.939895:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775386.939900:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775386.939905:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775386.939909:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775386.939914:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775386.939918:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775386.939921:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775386.939923:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775386.939926:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775386.939931:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775386.939935:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775386.939939:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775386.939946:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775386.939948:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775386.939950:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775386.939952:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775386.939955:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775386.939961:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775386.939965:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775386.939968:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775386.939973:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775386.939979:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775386.939983:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775386.939987:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775386.939993:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775386.939997:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775386.940004:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775775386.940008:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775386.940012:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775386.940015:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775386.940020:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775386.940023:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775386.940025:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775386.940032:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775775386.940038:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775386.940047:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710335744/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775386.940059:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775386.940063:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775386.940069:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710335744/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775386.940084:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775386.940090:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775386.940097:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:0.1:1775775386.940103:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775775386.940105:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775386.940111:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:0.1:1775775386.940111:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000100:00000040:2.0:1775775386.940121:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12279) 00000800:00000200:0.1:1775775386.940654:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775386.940656:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775775386.940996:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775386.940998:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00010000:00000001:3.0:1775775387.067284:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775387.067292:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775387.067299:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775387.067303:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775387.067306:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775387.067311:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775387.067314:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775387.067317:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775387.067320:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775387.067323:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775387.067327:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775387.067329:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775387.067331:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775387.067333:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775387.067339:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775387.067341:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775387.067343:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775387.067345:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775387.963255:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775387.963260:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775387.963261:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775387.963265:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775387.963269:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775387.963272:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775387.963274:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775387.963277:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775387.963279:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775388.091255:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775388.091260:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775388.091264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775388.091264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775388.091267:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775388.091268:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775388.987227:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775388.987232:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775388.987234:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775388.987238:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775388.987241:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775388.987244:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775388.987250:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775388.987252:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775388.987255:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775389.115251:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775389.115256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775389.115260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775389.115261:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775389.115264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775389.115265:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775389.115352:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775389.115358:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775389.115362:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775390.011202:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775390.011209:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775390.011210:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775390.011214:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775390.011217:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775390.011220:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775390.011222:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775390.011224:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775390.011227:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775390.139248:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775390.139253:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775390.139257:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775390.139258:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775390.139261:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775390.139262:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775391.035300:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775391.035311:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775391.035314:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775391.035321:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775391.035325:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775391.035330:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775391.035332:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775391.035335:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775391.035337:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775391.163252:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775391.163261:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775391.163269:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775391.163270:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775391.163274:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775391.163276:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775391.163306:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775391.163310:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775391.163315:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775392.059209:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775392.059217:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775392.059219:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775392.059223:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775392.059228:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775775392.059232:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1775775392.059237:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12274 00000100:00000001:3.0:1775775392.059238:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775392.059242:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775392.059247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000400:00000200:2.0:1775775392.059255:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775775392.059258:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775392.059264:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775775392.059268:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:3.0:1775775392.059270:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1775775392.059273:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1775775392.059275:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775775392.059281:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775392.059281:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775392.059288:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775392.059292:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775392.059295:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775392.059298:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775392.059299:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775392.059301:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775392.059304:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775392.059306:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775392.059308:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775392.059311:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775392.059313:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775392.059314:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775392.059316:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775392.059318:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775392.059320:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775392.059322:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775392.059324:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775392.059327:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775392.059336:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775392.059338:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775392.059340:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775392.059347:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775775392.059350:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775392.059353:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775392.059354:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775392.059356:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775392.059358:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775392.059362:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775392.059370:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775775392.059375:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775392.059382:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710335872/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775392.059395:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775392.059398:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775392.059401:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710335872/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775392.059418:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775392.059421:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775392.059423:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775392.059425:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775392.059427:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775392.059429:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775392.059432:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12274 00000400:00000200:2.0:1775775392.059437:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775392.059441:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775392.059444:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775392.059447:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775392.059449:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775392.059452:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775392.059454:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775392.059457:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775392.059460:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775392.059462:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775392.059466:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775392.059470:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775392.059473:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775392.059480:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775392.059486:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775392.059491:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775392.059496:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775392.059502:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775392.059509:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775392.059515:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775392.059519:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775392.059526:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775392.059530:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775392.059533:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775392.059538:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775392.059542:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775392.059544:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775392.059548:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775775392.059553:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775392.059559:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775392.059561:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775392.059566:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775392.059568:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775392.059571:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775392.059578:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775775392.059584:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775392.059593:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710336000/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775392.059619:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775392.059625:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775392.059635:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710336000/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775392.059653:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775392.059659:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775392.059666:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775392.059670:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775392.059673:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775392.059680:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775392.059688:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12274 00000400:00000200:2.0:1775775392.059698:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775392.059708:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775392.059714:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775392.059723:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775392.059727:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775392.059733:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775392.059735:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775392.059740:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775392.059742:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775392.059745:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:0.1:1775775392.059746:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000040:2.0:1775775392.059748:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775392.059762:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775392.059766:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:0.1:1775775392.059768:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000100:00000040:2.0:1775775392.059773:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775392.059779:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775392.059785:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775392.059788:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775392.059794:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775392.059803:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775392.059809:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775392.059815:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775392.059819:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775392.059828:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775392.059833:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775392.059836:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775392.059843:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775392.059847:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775392.059853:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775775392.059859:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775392.059865:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775392.059868:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775392.059871:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775392.059875:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775392.059878:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775392.059886:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775775392.059894:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775392.059903:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710336128/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775392.059912:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775392.059916:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775392.059926:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710336128/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775392.059949:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775392.059952:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775392.059959:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775392.059963:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775392.059969:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775392.059974:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12284) 00000800:00000200:0.1:1775775392.060182:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775392.060184:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775775392.060370:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775392.060372:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00010000:00000001:3.0:1775775392.187260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775392.187270:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775392.187272:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775392.187275:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775392.187281:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775392.187283:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775392.187285:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775392.187286:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775392.187290:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775392.187291:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775392.187294:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775392.187295:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775393.083268:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775393.083277:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775393.083279:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775393.083286:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775393.083292:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775393.083297:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775393.083300:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775393.083304:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775393.083307:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775393.211245:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775393.211255:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775393.211263:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775393.211264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775393.211269:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775393.211270:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775394.107205:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775394.107212:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775394.107214:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775394.107219:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775394.107223:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775394.107226:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775394.107229:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775394.107232:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775394.107235:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775394.235227:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775394.235232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775394.235237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775394.235238:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775394.235241:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775394.235242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775394.235309:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775394.235314:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775394.235318:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775395.131240:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775395.131247:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775395.131249:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775395.131254:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775395.131257:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775395.131261:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775395.131263:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775395.131265:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775395.131267:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775395.259202:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775395.259207:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775395.259210:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775395.259211:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775395.259214:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775395.259215:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775396.155220:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775396.155226:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775396.155228:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775396.155232:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775396.155235:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775396.155238:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775396.155240:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775396.155243:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775396.155245:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775396.283249:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775396.283254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775396.283258:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775396.283259:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775396.283261:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775396.283262:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775396.283282:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775396.283285:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775396.283287:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775397.179760:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775397.179764:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775397.179767:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775397.179771:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775397.179776:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775397.179780:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.1:1775775397.179780:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775775397.179786:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000800:00000200:1.1:1775775397.179787:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000100:00000001:3.0:1775775397.179788:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775397.179793:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775397.180291:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775397.180294:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775775397.180564:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775397.180567:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00010000:00000001:3.0:1775775397.307225:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775397.307230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775397.307236:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775397.307239:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775397.307242:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775397.307245:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775397.307248:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775397.307251:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775397.307253:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775397.307256:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775397.307260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775397.307262:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775397.307263:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775397.307265:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775397.307267:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775397.307269:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775397.307271:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775397.307273:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775398.203247:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775398.203253:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775398.203255:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775398.203260:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775398.203263:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775398.203266:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775398.203268:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775398.203270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775398.203272:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775398.331217:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775398.331222:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775398.331226:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775398.331227:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775398.331230:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775398.331231:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775399.227220:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775399.227227:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775399.227229:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775399.227235:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775399.227239:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775399.227243:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775399.227246:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775399.227248:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775399.227251:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775399.355236:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775399.355240:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775399.355244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775399.355245:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775399.355248:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775399.355249:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775399.355302:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775399.355306:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775399.355308:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775400.251245:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775400.251253:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775400.251255:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775400.251261:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775400.251265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775400.251268:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775400.251271:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775400.251273:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775400.251275:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775400.379213:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775400.379218:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775400.379222:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775400.379223:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775400.379226:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775400.379227:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775401.275218:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775401.275227:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775401.275230:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775401.275237:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775401.275243:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775401.275247:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775401.275251:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775401.275254:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775401.275258:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775401.403282:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775401.403291:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775401.403295:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775401.403298:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775401.403304:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775401.403307:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775401.403309:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775401.403312:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775401.403326:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775401.403328:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775401.403332:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775401.403333:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775401.403366:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775401.403369:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775401.403373:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775402.299258:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775402.299269:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775402.299274:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775402.299283:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775402.299291:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775402.299297:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775402.299305:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775402.299313:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775402.299321:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775775402.299920:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775402.299928:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775775402.300449:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775402.300452:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1775775402.300696:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775402.300698:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00010000:00000001:3.0:1775775402.427210:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775402.427216:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775402.427222:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775402.427223:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775402.427226:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775402.427228:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775403.323238:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775403.323244:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775403.323246:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775403.323250:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775403.323253:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775403.323256:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775403.323258:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775403.323261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775403.323263:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775403.451262:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775403.451268:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775403.451274:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775403.451276:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775403.451279:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775403.451281:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775404.347205:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775404.347212:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775404.347214:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775404.347219:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775404.347223:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775404.347226:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775404.347229:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775404.347231:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775404.347234:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775404.475186:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775404.475190:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775404.475195:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775404.475196:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775404.475199:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775404.475200:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775404.475326:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775404.475331:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775404.475335:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775405.371246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775405.371255:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775405.371258:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775405.371265:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775405.371270:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775405.371273:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775405.371277:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775405.371280:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775405.371284:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775405.499274:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775405.499280:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775405.499286:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775405.499288:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775405.499292:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775405.499294:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775406.395210:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775406.395216:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775406.395217:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775406.395221:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775406.395224:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775406.395227:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775406.395230:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775406.395232:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775406.395234:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775406.523254:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775406.523258:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775406.523263:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775406.523264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775406.523267:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775406.523268:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775406.523289:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775406.523292:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775406.523295:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775407.419203:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775407.419209:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775407.419211:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775407.419216:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775407.419219:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775407.419222:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775407.419224:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775407.419226:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775407.419229:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775407.419783:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775407.419789:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775775407.420275:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775407.420278:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775775407.420611:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775407.420614:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00010000:00000001:3.0:1775775407.547208:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775407.547213:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775407.547218:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775407.547221:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775407.547222:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775407.547226:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775407.547228:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775407.547230:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775407.547231:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775407.547233:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775407.547236:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775407.547237:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775407.547237:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775407.547238:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775407.547239:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775407.547240:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775407.547241:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775407.547242:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775408.443291:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775408.443299:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775408.443301:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775408.443307:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775408.443311:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775408.443315:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775408.443317:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775408.443319:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775408.443321:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775408.571239:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775408.571243:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775408.571247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775408.571248:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775408.571251:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775408.571252:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775409.467228:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775409.467233:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775409.467235:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775409.467239:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775409.467242:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775409.467245:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775409.467247:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775409.467249:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775409.467251:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775409.595263:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775409.595270:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775409.595276:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775409.595278:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775409.595282:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775409.595283:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775409.595353:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775409.595357:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775409.595361:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775410.491257:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775410.491263:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775410.491265:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775410.491269:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775410.491271:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775410.491274:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775410.491276:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775410.491278:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775410.491280:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775410.619219:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775410.619224:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775410.619228:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775410.619229:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775410.619231:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775410.619232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775411.515205:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775411.515210:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775411.515212:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775411.515216:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775411.515220:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775411.515222:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775411.515225:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775411.515227:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775411.515229:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775411.643225:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775411.643230:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775411.643232:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775411.643235:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775411.643239:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775411.643241:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775411.643242:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775411.643244:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775411.643247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775411.643249:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775411.643252:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775411.643252:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775411.643270:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775411.643272:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775411.643274:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775412.539222:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775412.539229:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775412.539232:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775412.539237:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775412.539241:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775412.539249:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12295 00000400:00000200:3.0:1775775412.539259:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775412.539267:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775412.539270:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775412.539273:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775412.539276:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775412.539278:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775412.539280:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775412.539283:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775412.539285:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775412.539287:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775412.539289:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775412.539291:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775412.539292:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775412.539295:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775412.539296:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775412.539297:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775412.539298:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775412.539300:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775412.539302:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775412.539304:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775412.539307:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775412.539309:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775412.539311:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775412.539313:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775412.539314:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775412.539321:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775775412.539323:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775412.539326:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775412.539328:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775412.539330:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775412.539338:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775412.539342:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775412.539351:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000001:3.0:1775775412.539357:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775412.539364:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710337408/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775412.539371:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775412.539375:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775412.539378:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710337408/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775412.539413:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775412.539415:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775412.539418:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775412.539419:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775412.539420:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775412.539423:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775412.539425:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12295 00000400:00000200:3.0:1775775412.539428:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775412.539431:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775412.539433:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775412.539435:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775412.539437:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775412.539440:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775412.539442:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775412.539444:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775412.539445:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775412.539446:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775412.539447:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775412.539450:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775412.539452:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775412.539456:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775412.539459:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775412.539464:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775412.539469:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775412.539476:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775412.539483:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775412.539488:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775412.539494:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775412.539502:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775412.539508:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775412.539513:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775412.539518:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775412.539525:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775412.539531:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775412.539536:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775775412.539539:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775412.539541:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775412.539544:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775412.539546:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775412.539547:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775412.539549:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775412.539557:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000001:3.0:1775775412.539563:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775412.539571:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710337536/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775412.539585:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775412.539588:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775412.539594:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710337536/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775412.539606:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775412.539611:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775412.539616:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775412.539619:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775412.539621:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775412.539625:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775412.539631:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12295 00000400:00000200:3.0:1775775412.539641:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775412.539649:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775412.539654:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775412.539661:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775412.539668:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775412.539674:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775412.539678:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775412.539682:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775412.539684:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775412.539687:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775412.539691:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775412.539696:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:1.1:1775775412.539696:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:3.0:1775775412.539702:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:1.1:1775775412.539702:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000100:00000040:3.0:1775775412.539711:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775412.539716:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775412.539720:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775412.539727:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775412.539731:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775412.539739:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775412.539744:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775412.539747:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775412.539751:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775412.539756:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775412.539760:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775412.539763:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775412.539766:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775412.539767:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775412.539770:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775775412.539772:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775412.539773:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775412.539774:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775412.539777:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775412.539779:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775412.539782:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775412.539788:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000100:00000001:3.0:1775775412.539796:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775412.539808:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710337664/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775412.539821:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775412.539825:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775412.539832:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710337664/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775412.539853:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775412.539858:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775412.539863:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775412.539867:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775412.539873:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775412.539878:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12305) 00000800:00000200:1.1:1775775412.540067:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775412.540068:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775775412.540356:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775412.540358:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00010000:00000001:3.0:1775775412.667238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775412.667243:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775412.667248:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775412.667250:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775412.667254:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775412.667256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775775413.371229:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775775413.371235:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775775413.371239:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00010000:00000001:3.0:1775775413.691238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775413.691244:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775413.691249:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775413.691251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775413.691254:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775413.691256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775414.715240:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775414.715245:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775414.715251:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775414.715252:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775414.715256:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775414.715257:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775414.715313:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775414.715316:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775414.715318:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775415.739222:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775415.739227:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775415.739230:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775415.739231:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775415.739234:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775415.739235:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775416.763209:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775416.763216:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775416.763227:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775416.763228:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775416.763232:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775416.763234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775416.763257:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775416.763260:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775416.763263:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1775775417.659837:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775417.659843:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775775417.660164:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775417.660166:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775775417.660407:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775417.660408:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00010000:00000001:3.0:1775775417.787220:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775417.787225:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775417.787229:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775417.787232:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775417.787234:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775417.787236:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775417.787238:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775417.787240:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775417.787241:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775417.787243:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775417.787246:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775417.787247:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775417.787248:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775417.787249:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775417.787250:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775417.787250:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775417.787251:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775417.787253:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775418.811237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775418.811241:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775418.811247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775418.811247:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775418.811250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775418.811251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775419.835202:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775419.835207:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775419.835211:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775419.835212:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775419.835215:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775419.835216:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775419.835283:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775419.835287:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775419.835291:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775420.859245:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775420.859274:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775420.859279:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775420.859280:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775420.859283:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775420.859284:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775421.883205:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775421.883211:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775421.883213:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775421.883216:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775421.883219:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775421.883221:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775421.883223:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775421.883224:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775421.883227:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775421.883228:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775421.883244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775421.883245:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775421.883265:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775421.883268:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775421.883270:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775422.779283:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775422.779291:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775422.779296:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775422.779300:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775422.779305:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775422.779312:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12305 00000400:00000200:2.0:1775775422.779320:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775422.779326:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775422.779329:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775422.779331:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775422.779334:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775422.779335:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775422.779338:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775422.779342:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775422.779343:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775422.779346:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775422.779349:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775422.779352:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775422.779355:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775422.779358:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775422.779360:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775422.779362:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775422.779364:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775422.779368:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775422.779372:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775422.779374:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775422.779378:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775422.779381:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775422.779384:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775422.779386:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775422.779389:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775422.779398:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775775422.779401:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775422.779405:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775422.779407:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775422.779409:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775422.779412:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775422.779416:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775422.779426:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775775422.779434:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775422.779444:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710338176/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775422.779455:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775422.779459:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775422.779463:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710338176/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775422.779479:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775422.779481:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775422.779483:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775422.779485:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775422.779486:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775422.779488:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775422.779489:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12305 00000400:00000200:2.0:1775775422.779493:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775422.779495:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775422.779497:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775422.779499:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775422.779502:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775422.779503:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775422.779505:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775422.779506:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775422.779507:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775422.779508:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775422.779510:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775422.779513:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775422.779518:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775422.779536:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775422.779542:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775422.779546:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775422.779553:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775422.779560:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775422.779568:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775422.779574:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775422.779577:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775422.779584:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775422.779587:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775422.779589:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775422.779593:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775422.779596:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775422.779601:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775422.779605:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775775422.779608:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775422.779613:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775422.779615:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775422.779618:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775422.779619:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775422.779621:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775422.779627:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775775422.779633:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775422.779640:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710338304/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775422.779656:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775422.779660:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775422.779669:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710338304/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775422.779687:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775422.779693:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775422.779699:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775422.779701:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775422.779702:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775422.779708:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775422.779712:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12305 00000400:00000200:2.0:1775775422.779716:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775422.779722:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775422.779734:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775422.779742:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775422.779746:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775422.779752:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775422.779754:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775422.779759:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775422.779761:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775422.779765:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775422.779768:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775422.779771:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775422.779772:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775422.779777:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775422.779782:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775422.779787:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775422.779792:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775422.779795:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775422.779799:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775422.779804:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775422.779805:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775422.779808:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775422.779811:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775422.779814:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775422.779815:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775422.779817:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775422.779819:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775422.779824:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775775422.779828:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775422.779830:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775422.779832:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775422.779836:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775422.779838:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775422.779840:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775775422.779841:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000010:2.0:1775775422.779844:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000800:00000200:0.1:1775775422.779849:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1775775422.779854:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775422.779864:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710338432/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775422.779880:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775422.779884:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775422.779892:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710338432/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775422.779927:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775422.779935:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775422.779943:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775422.779950:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775422.779955:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775422.779963:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12315) 00000800:00000200:0.1:1775775422.780244:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775422.780246:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775775422.780475:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775422.780476:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00010000:00000001:3.0:1775775422.907270:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775422.907276:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775422.907282:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775422.907283:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775422.907287:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775422.907288:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775423.931242:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775423.931247:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775423.931253:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775423.931254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775423.931258:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775423.931259:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775424.955303:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775424.955311:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775424.955319:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775424.955321:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775424.955325:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775424.955327:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775424.955424:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775424.955431:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775424.955436:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775425.979236:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775425.979240:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775425.979244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775425.979245:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775425.979248:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775425.979249:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775427.003275:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775427.003281:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775427.003288:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775427.003289:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775427.003293:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775427.003295:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775427.003322:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775427.003325:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775427.003328:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775427.899209:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775427.899216:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775427.899218:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775427.899222:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775427.899227:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775427.899235:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12310 00000400:00000200:2.0:1775775427.899246:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775427.899252:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775427.899269:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775427.899272:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775427.899275:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775427.899278:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775427.899280:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775427.899283:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775427.899285:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775427.899287:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775427.899289:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775427.899291:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775427.899292:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775427.899294:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775427.899296:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775427.899297:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775427.899299:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775427.899301:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775427.899303:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775427.899305:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775427.899307:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775427.899309:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775427.899311:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775427.899312:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775427.899314:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775427.899323:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775775427.899325:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775427.899331:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775427.899333:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775427.899335:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775427.899337:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775427.899340:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775427.899346:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775427.899351:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775427.899358:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710338560/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775427.899367:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775427.899370:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775427.899374:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710338560/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775427.899390:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775427.899392:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775427.899394:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775427.899396:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775427.899397:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775427.899399:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775427.899400:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12310 00000400:00000200:2.0:1775775427.899403:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775427.899405:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775427.899407:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775427.899409:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775427.899410:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775427.899412:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775427.899415:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775427.899417:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775427.899420:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775427.899422:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775427.899424:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775427.899427:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775427.899429:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775427.899433:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775427.899436:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775427.899441:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775427.899447:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775427.899451:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775427.899458:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775427.899464:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775427.899469:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775427.899476:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775427.899482:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775427.899484:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775427.899490:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775427.899495:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775427.899496:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775427.899503:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775775427.899505:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775427.899511:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775427.899513:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775427.899517:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775427.899518:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775427.899521:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775427.899527:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775427.899532:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775427.899540:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710338688/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775427.899556:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775427.899560:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775427.899567:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710338688/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775427.899582:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775427.899587:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775427.899591:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775427.899594:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775427.899596:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775427.899598:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775427.899602:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12310 00000400:00000200:2.0:1775775427.899609:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775427.899614:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775427.899619:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775427.899624:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775427.899626:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775427.899630:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775427.899631:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775427.899635:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775427.899638:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775427.899640:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775427.899641:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775427.899645:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775427.899648:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775427.899654:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775427.899656:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775427.899660:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775427.899662:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775427.899666:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775427.899668:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775427.899671:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775427.899675:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775427.899680:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775427.899684:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775427.899688:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775427.899691:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775427.899696:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775427.899699:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775427.899705:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775775427.899710:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775427.899716:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775427.899718:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775427.899723:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775427.899726:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775427.899729:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775427.899733:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775775427.899738:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775427.899745:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710338816/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775427.899758:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775427.899762:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775427.899771:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710338816/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775427.899786:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775427.899792:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775427.899798:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775427.899804:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.1:1775775427.899810:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775775427.899812:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:0.1:1775775427.899818:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00000040:2.0:1775775427.899819:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12320) 00000800:00000200:0.1:1775775427.900337:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775427.900339:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775775427.900649:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775427.900652:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00010000:00000001:3.0:1775775428.027247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775428.027253:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775428.027257:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775428.027260:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775428.027262:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775428.027264:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775428.027266:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775428.027268:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775428.027270:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775428.027271:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775428.027274:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775428.027275:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775428.027276:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775428.027276:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775428.027277:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775428.027278:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775428.027279:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775428.027280:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775429.051208:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775429.051212:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775429.051217:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775429.051217:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775429.051220:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775429.051221:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775430.075237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775430.075242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775430.075246:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775430.075247:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775430.075250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775430.075250:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775430.075320:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775430.075324:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775430.075327:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775431.099241:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775431.099246:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775431.099250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775431.099251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775431.099264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775431.099265:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775432.123274:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775432.123283:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775432.123286:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775432.123291:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775432.123297:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775432.123300:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775432.123302:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775432.123305:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775432.123310:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775432.123312:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775432.123317:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775432.123319:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775432.123357:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775432.123360:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775432.123363:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775433.019258:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775433.019278:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775433.019282:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775433.019287:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775433.019293:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775433.019302:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12315 00000400:00000200:2.0:1775775433.019314:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775433.019321:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775433.019326:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775433.019331:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775433.019335:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775433.019339:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775433.019342:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775433.019348:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775433.019350:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775433.019352:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775433.019364:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775433.019366:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775433.019369:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775433.019372:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775433.019373:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775433.019375:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775433.019377:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775433.019383:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775433.019387:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775433.019389:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775433.019393:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775433.019396:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775433.019399:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775433.019402:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775433.019404:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775433.019414:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775775433.019417:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775433.019422:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775433.019424:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775433.019427:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775433.019430:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775433.019434:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775433.019441:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775433.019446:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775433.019454:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710338944/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775433.019462:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775433.019465:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775433.019468:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710338944/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775433.019490:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775433.019493:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775433.019495:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775433.019497:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775433.019498:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775433.019500:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775433.019502:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12315 00000400:00000200:2.0:1775775433.019505:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775433.019507:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775433.019509:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775433.019511:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775433.019512:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775433.019514:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775433.019515:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775433.019517:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775433.019518:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775433.019519:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775433.019520:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775433.019522:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775433.019523:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775433.019526:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775433.019527:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775433.019529:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775433.019531:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775433.019533:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775433.019535:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775433.019537:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775433.019538:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775433.019543:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775433.019548:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775433.019550:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775433.019555:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775433.019558:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775433.019560:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775433.019562:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775775433.019566:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775433.019570:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775433.019573:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775433.019576:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775433.019579:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775433.019582:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775433.019585:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775775433.019588:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775433.019595:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710339072/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775433.019608:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775433.019611:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775433.019618:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710339072/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775433.019633:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775433.019639:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775433.019643:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775433.019647:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775433.019651:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775433.019655:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775433.019661:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12315 00000400:00000200:2.0:1775775433.019669:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775433.019673:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775433.019677:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775433.019681:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775433.019684:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775433.019689:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775433.019691:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775433.019694:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775433.019697:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775433.019699:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775433.019701:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775433.019705:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775433.019707:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775433.019715:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775433.019719:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775433.019722:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775433.019727:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775433.019732:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775433.019740:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775433.019746:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775433.019750:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775433.019755:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775433.019761:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775433.019764:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775433.019766:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775433.019772:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775433.019776:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775433.019781:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775775433.019786:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775433.019791:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775433.019793:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775433.019804:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775433.019807:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775433.019809:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775433.019816:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775775433.019822:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775433.019828:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710339200/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775433.019843:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775433.019848:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775433.019857:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710339200/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775433.019870:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775433.019876:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775433.019883:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:0.1:1775775433.019889:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775775433.019891:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.1:1775775433.019896:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000100:00000040:2.0:1775775433.019897:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775433.019906:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12325) 00000800:00000200:0.1:1775775433.020359:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775433.020361:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775775433.020671:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775433.020674:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000100:00000001:0.0:1775775433.020916:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775433.020918:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775433.147307:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775433.147316:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775433.147324:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775433.147326:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775433.147330:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775433.147332:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775434.043252:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775775434.043258:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775434.043260:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775434.043265:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775775434.043268:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775434.043271:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775434.043273:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775434.043276:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775434.043278:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775434.171198:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775434.171203:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775434.171207:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775434.171208:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775434.171211:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775434.171212:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775435.067210:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775775435.067216:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775435.067218:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775435.067224:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775775435.067229:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775435.067232:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775435.067235:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775435.067238:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775435.067240:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775435.195229:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775435.195234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775435.195238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775435.195239:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775435.195242:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775435.195243:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775435.195305:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775435.195310:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775435.195313:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775775436.091219:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775775436.091225:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775436.091227:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775436.091231:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775775436.091234:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775436.091238:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775436.091240:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775436.091242:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775436.091244:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775436.219222:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775436.219228:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775436.219233:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775436.219235:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775436.219238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775436.219240:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775436.923240:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4306991878, at 4306991936 00000100:00000010:0.0:1775775436.923256:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775437.115241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775775437.115246:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775437.115248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775437.115253:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775775437.115255:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775437.115259:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775437.115262:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775437.115266:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775437.115273:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775437.243237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775437.243242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775437.243246:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775437.243247:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775437.243250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775437.243251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775437.243270:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775437.243272:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775437.243275:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775775438.139206:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775775438.139211:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775438.139213:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775438.139218:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775775438.139221:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775438.139224:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775438.139226:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775438.139229:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775438.139231:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775438.139234:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775438.139240:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775438.139242:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775438.139245:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775438.139249:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775438.139255:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12320 00000400:00000200:3.0:1775775438.139263:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775438.139268:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775438.139270:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775438.139273:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775438.139275:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775438.139277:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775438.139285:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775438.139288:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775438.139290:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775438.139292:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775438.139294:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775438.139296:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775438.139298:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775438.139300:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775438.139302:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775438.139303:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775438.139304:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775438.139307:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775438.139309:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775438.139310:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775438.139313:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775438.139315:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775438.139316:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775438.139318:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775438.139319:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775438.139327:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775775438.139330:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775438.139333:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775438.139334:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775438.139336:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775438.139339:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775438.139342:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775438.139353:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775775438.139359:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775438.139366:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710339328/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775438.139374:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775438.139378:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775438.139381:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710339328/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775438.139396:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775438.139398:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775438.139401:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775438.139402:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775438.139404:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775438.139406:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775438.139408:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12320 00000400:00000200:3.0:1775775438.139411:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775438.139413:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775438.139415:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775438.139416:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775438.139418:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775438.139420:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775438.139421:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775438.139422:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775438.139423:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775438.139424:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775438.139426:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775438.139428:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775438.139430:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775438.139433:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775438.139434:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775438.139436:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775438.139437:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:0.0:1775775438.139438:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1775775438.139439:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000001:0.0:1775775438.139443:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775775438.139443:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775438.139447:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775438.139448:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1775775438.139448:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775438.139454:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775438.139456:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775438.139457:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71500 x1862022710339328/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775438.139457:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775438.139462:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775438.139467:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775438.139471:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775438.139477:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000100:00000001:0.0:1775775438.139478:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1775775438.139481:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000001:0.0:1775775438.139482:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775438.139485:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775438.139487:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775438.139488:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1775775438.139488:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775438.139491:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775438.139492:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775438.139493:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775438.139497:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775438.139498:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000100:00100000:0.0:1775775438.139502:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710339328:192.168.201.115@tcp:250:kworker.0 00000100:00000001:3.0:1775775438.139503:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775438.139509:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775438.139512:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:3.0:1775775438.139512:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710339456/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1775775438.139515:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775438.139519:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775775438.139524:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000010:0.0:1775775438.139526:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 00000100:00000001:3.0:1775775438.139527:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775438.139529:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775438.139535:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710339456/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775775438.139536:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775438.139548:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000100:00000001:3.0:1775775438.139549:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775438.139554:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:0.0:1775775438.139556:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00080000:3.0:1775775438.139561:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775438.139564:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:0.0:1775775438.139566:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710339328, portal 25 00000100:00080000:3.0:1775775438.139566:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775438.139571:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775775438.139572:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775775438.139573:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849009408 : -123823860542208 : ffff8f6202c71500) 00000100:00080000:3.0:1775775438.139578:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12320 00000100:00000040:0.0:1775775438.139580:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71500 x1862022710339328/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775775438.139588:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775438.139594:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1775775438.139594:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775438.139598:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775438.139600:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775438.139603:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000040:3.0:1775775438.139604:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775438.139607:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775438.139608:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775438.139609:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775438.139610:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775438.139612:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775438.139613:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000200:0.0:1775775438.139614:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710339328, offset 0 00000020:00000040:3.0:1775775438.139614:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775438.139619:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:0.0:1775775438.139623:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000020:00000001:3.0:1775775438.139628:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:0.0:1775775438.139633:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775775438.139636:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775438.139638:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775775438.139641:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000040:3.0:1775775438.139642:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775438.139646:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775438.139648:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:0.0:1775775438.139649:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:3.0:1775775438.139654:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775438.139658:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775438.139659:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1775775438.139661:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 02000000:00000001:3.0:1775775438.139662:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775438.139665:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775438.139667:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775438.139668:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1775775438.139671:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000008:00000001:3.0:1775775438.139671:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775438.139676:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:3.0:1775775438.139676:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775438.139681:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000400:00000200:0.0:1775775438.139682:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453203:453203:256:4294967295] 192.168.201.115@tcp LPNI seq info [453128:453128:8:4294967295] 00000020:00000040:3.0:1775775438.139685:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:0.0:1775775438.139689:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775775438.139689:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775438.139691:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775438.139693:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775438.139694:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775438.139696:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775438.139699:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775438.139705:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 02000000:00000010:3.0:1775775438.139705:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000100:00000001:3.0:1775775438.139711:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775775438.139713:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cb00. 00000100:00100000:3.0:1775775438.139717:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710339584/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.0:1775775438.139724:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775438.139729:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000100:00000001:3.0:1775775438.139729:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775438.139734:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775775438.139737:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775775438.139739:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710339584/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1775775438.139741:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775775438.139752:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775775438.139753:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775775438.139755:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cb00. 00000100:00000040:3.0:1775775438.139756:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775438.139760:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:0.0:1775775438.139762:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:3.0:1775775438.139762:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775438.139767:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:0.0:1775775438.139771:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000040:3.0:1775775438.139774:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12330) 00000400:00000200:0.0:1775775438.139775:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775775438.139778:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775775438.139782:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775438.139785:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71500 x1862022710339328/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.139793:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775438.139795:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71500 x1862022710339328/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775463 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.139805:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775438.139806:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775438.139811:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775438.139813:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775438.139815:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4306996998 00000100:00000010:0.0:1775775438.139819:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775438.139825:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775438.139830:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4306996998 00000100:00000010:0.0:1775775438.139832:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c34e0. 00000100:00000001:0.0:1775775438.139834:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775438.139836:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775438.139838:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775438.139840:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775438.139843:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775438/real 1775775438] req@ffff8f6202c71500 x1862022710339328/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775438.139849:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775438.139850:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775438.139851:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775438.139853:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775438.139854:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775438.139856:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775438.139858:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71500 x1862022710339328/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775438.139864:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71500 x1862022710339328/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.139868:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775438.139869:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775438.139871:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775438.139872:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775438.139875:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71500 x1862022710339328/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.139880:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775438.139883:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71500 x1862022710339328/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.139887:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775438.139889:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775438.139890:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775438.139891:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775438.139892:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775438.139894:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775438.139895:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775438.139898:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775438.139899:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775438.139902:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71500 x1862022710339328/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775438.139907:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710339328:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775438.139910:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775438.139911:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775438.139913:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71500 x1862022710339328/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.139917:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775438.139919:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775438.139921:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775438.139923:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775438.139924:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775438.139925:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775438.139926:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775438.139928:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34c00. 02000000:00000001:0.0:1775775438.139931:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775438.139933:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775438.139934:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71500. 00000100:00000001:0.0:1775775438.139937:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775438.139937:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775438.139939:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775438.139945:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775438.139958:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775438.139960:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775438.139961:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775438.139962:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775438.139964:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70700 x1862022710339456/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.139970:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775438.139971:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775438.139972:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775438.139973:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775438.139975:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775438.139976:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710339456:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775438.139978:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775438.139980:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775438.139981:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775438.139982:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775438.139984:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775438.139985:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775438.139988:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775438.139990:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775438.139992:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775438.139994:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710339456, portal 10 00000100:00000001:0.0:1775775438.139995:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775438.139996:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849005824 : -123823860545792 : ffff8f6202c70700) 00000100:00000040:0.0:1775775438.140001:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70700 x1862022710339456/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.140005:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775438.140007:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775438.140009:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775775438.140010:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710339456, offset 0 00000400:00000200:0.0:1775775438.140013:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775438.140016:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775438.140018:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775438.140020:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775438.140022:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775438.140024:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775438.140026:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775438.140028:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453204:453204:256:4294967295] 192.168.201.115@tcp LPNI seq info [453129:453129:8:4294967295] 00000400:00000200:0.0:1775775438.140031:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775438.140035:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775438.140038:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775438.140039:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cb00. 00000800:00000200:0.0:1775775438.140042:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775438.140044:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775775438.140047:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775438.140048:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775775438.140056:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775438.140058:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cb00. 00000400:00000200:0.0:1775775438.140060:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775438.140062:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775438.140064:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775775438.140065:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775775438.140067:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775438.140069:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70700 x1862022710339456/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.140074:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775438.140076:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70700 x1862022710339456/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775463 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.140081:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775438.140082:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775438.140084:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775438.140087:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775438.140089:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775438.140090:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4306993163 00000100:00000001:0.0:1775775438.140092:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4306996998 00000100:00000001:0.0:1775775438.140094:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4306993163 00000100:00000001:0.0:1775775438.140095:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4306996998 00000100:00000001:0.0:1775775438.140097:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775438.140098:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775438.140099:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775438.140102:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775438/real 1775775438] req@ffff8f6202c70700 x1862022710339456/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775438.140106:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775438.140107:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775438.140108:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775438.140109:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775438.140110:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775438.140111:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775438.140113:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70700 x1862022710339456/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775438.140119:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70700 x1862022710339456/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.140123:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775438.140124:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775438.140125:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775438.140126:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775438.140128:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70700 x1862022710339456/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.140132:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775438.140151:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70700 x1862022710339456/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.140156:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775438.140158:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775438.140159:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775438.140160:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775438.140161:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775438.140164:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775438.140166:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775438.140168:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775438.140169:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775438.140172:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70700 x1862022710339456/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775438.140177:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710339456:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775438.140179:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775438.140181:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70a80 x1862022710339584/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.140185:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775438.140186:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775438.140188:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775438.140189:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775438.140190:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775438.140192:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710339584:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775438.140194:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775438.140195:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775438.140196:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775438.140197:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775438.140200:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775775438.140201:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775438.140203:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775438.140205:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775438.140207:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775438.140209:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710339584, portal 10 00000100:00000001:0.0:1775775438.140210:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775438.140211:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849006720 : -123823860544896 : ffff8f6202c70a80) 00000100:00000040:0.0:1775775438.140215:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70a80 x1862022710339584/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.140220:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775438.140221:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775438.140223:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775775438.140224:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710339584, offset 0 00000400:00000200:0.0:1775775438.140227:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775438.140229:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775438.140231:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775438.140233:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775438.140235:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775438.140237:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775438.140239:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775438.140241:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453205:453205:256:4294967295] 192.168.201.115@tcp LPNI seq info [453130:453130:8:4294967295] 00000400:00000200:0.0:1775775438.140244:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775438.140248:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775438.140250:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775438.140252:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cb00. 00000800:00000200:0.0:1775775438.140255:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775438.140257:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775775438.140260:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775438.140261:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775775438.140264:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775438.140265:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cb00. 00000400:00000200:0.0:1775775438.140267:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775438.140269:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775438.140272:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775775438.140273:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775775438.140275:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775438.140277:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70a80 x1862022710339584/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.140283:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775438.140285:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70a80 x1862022710339584/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775463 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.140289:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775438.140290:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775438.140292:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775438.140294:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775438.140296:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775438.140297:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4306993164 00000100:00000001:0.0:1775775438.140299:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4306996998 00000100:00000001:0.0:1775775438.140300:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4306993164 00000100:00000001:0.0:1775775438.140302:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4306996998 00000100:00000001:0.0:1775775438.140303:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775438.140304:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775438.140305:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775438.140307:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775438/real 1775775438] req@ffff8f6202c70a80 x1862022710339584/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775438.140312:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775438.140313:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775438.140315:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775438.140316:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775438.140317:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775438.140319:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775438.140320:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70a80 x1862022710339584/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775438.140325:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70a80 x1862022710339584/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.140330:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775438.140331:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775438.140332:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775438.140333:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775438.140335:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70a80 x1862022710339584/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.140339:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775438.140341:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70a80 x1862022710339584/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.140345:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775438.140346:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775438.140347:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775438.140348:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775438.140349:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775438.140350:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775438.140352:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775438.140354:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775438.140355:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775438.140357:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70a80 x1862022710339584/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775438.140362:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710339584:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775438.140364:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775438.140365:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775438.140367:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70700 x1862022710339456/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.140371:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775438.140373:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775438.140374:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775438.140376:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775438.140377:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775438.140378:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775438.140379:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775438.140380:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36000. 02000000:00000001:0.0:1775775438.140382:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775438.140383:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775438.140384:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70700. 00000100:00000001:0.0:1775775438.140385:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775438.140386:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775438.140387:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775438.140389:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70a80 x1862022710339584/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775463 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775438.140393:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775438.140394:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775438.140394:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775775438.140396:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775438.140396:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775438.140397:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775438.140398:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775438.140399:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36400. 02000000:00000001:0.0:1775775438.140400:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775438.140401:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775438.140401:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70a80. 00000100:00000001:0.0:1775775438.140402:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775438.140403:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775438.140404:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775438.140406:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775438.140407:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775438.140408:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775438.140409:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775438.267201:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775438.267205:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775438.267209:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775438.267212:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775438.267213:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775438.267216:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775438.267218:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775438.267220:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775438.267222:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775438.267223:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775438.267226:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775438.267227:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775438.267227:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775438.267228:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775438.267229:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775438.267230:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775438.267231:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775438.267233:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775439.291218:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775439.291224:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775439.291229:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775439.291230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775439.291233:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775439.291234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775440.315222:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775440.315227:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775440.315231:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775440.315232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775440.315235:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775440.315235:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775440.315296:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775440.315301:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775440.315305:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775441.339243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775441.339250:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775441.339253:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775441.339256:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775441.339261:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775441.339263:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775441.339265:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775441.339268:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775441.339272:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775441.339274:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775441.339278:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775441.339279:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775442.043221:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4306996998, at 4306997056 00000100:00000010:0.0:1775775442.043244:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00010000:00000001:3.0:1775775442.363235:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775442.363242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775442.363253:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775442.363255:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775442.363259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775442.363260:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775442.363284:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775442.363287:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775442.363290:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775775443.259564:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775443.259570:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775443.259575:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775443.259585:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234919f80 x1862022710339712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.259604:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775443.259607:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775443.259613:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775443.259617:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775443.259621:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775443.259625:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234919f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710339712:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775443.259630:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775443.259633:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775443.259636:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775443.259639:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775443.259645:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775775443.259648:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775443.259657:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775443.259662:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775443.259669:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775443.259676:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710339712, portal 25 00000100:00000001:0.0:1775775443.259680:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775443.259682:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684366720 : -123823025184896 : ffff8f6234919f80) 00000100:00000040:0.0:1775775443.259687:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234919f80 x1862022710339712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.259697:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775443.259703:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775443.259708:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775775443.259712:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710339712, offset 0 00000400:00000200:0.0:1775775443.259721:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775443.259730:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775443.259738:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775443.259743:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775443.259749:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775443.259756:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775443.259760:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775443.259767:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453206:453206:256:4294967295] 192.168.201.115@tcp LPNI seq info [453131:453131:8:4294967295] 00000400:00000200:0.0:1775775443.259774:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775443.259784:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775443.259792:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775443.259799:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cb00. 00000800:00000200:0.0:1775775443.259806:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775443.259811:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775775443.259821:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775443.259826:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1775775443.259833:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775443.259836:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cb00. 00000400:00000200:0.0:1775775443.259841:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775443.259848:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775443.259854:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775775443.259878:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775775443.259884:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775443.259889:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234919f80 x1862022710339712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.259904:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775443.259910:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234919f80 x1862022710339712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775468 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.259923:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775443.259926:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775443.259931:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775443.259941:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775443.259944:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307002118 00000100:00000010:0.0:1775775443.259950:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775443.259957:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775443.259965:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307002118 00000100:00000010:0.0:1775775443.259970:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3d80. 00000100:00000001:0.0:1775775443.259976:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775443.259980:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775443.259982:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775443.259986:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775443.259992:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775443/real 1775775443] req@ffff8f6234919f80 x1862022710339712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775443.260010:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775443.260012:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775443.260016:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775443.260018:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775443.260020:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775443.260023:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775443.260028:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234919f80 x1862022710339712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775443.260046:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234919f80 x1862022710339712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.260055:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775443.260058:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775443.260061:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775443.260064:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775443.260068:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234919f80 x1862022710339712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.260080:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775443.260084:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234919f80 x1862022710339712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.260090:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775443.260092:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775443.260093:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775443.260095:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775443.260096:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775443.260105:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775443.260107:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775443.260110:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775443.260112:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775443.260115:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234919f80 x1862022710339712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775443.260120:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234919f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710339712:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775443.260123:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775443.260125:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775443.260126:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234919f80 x1862022710339712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.260130:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775443.260168:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775443.260171:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775775443.260173:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775443.260175:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775443.260176:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775443.260177:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775443.260179:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4d400. 02000000:00000001:0.0:1775775443.260181:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775443.260182:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775443.260183:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234919f80. 00000100:00000001:0.0:1775775443.260186:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775443.260187:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775443.260188:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775443.260193:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775443.260194:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775443.260195:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775443.260196:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775443.260198:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775443.260200:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918380 x1862022710339840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.260209:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775443.260210:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775443.260212:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775443.260213:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775443.260214:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775443.260216:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710339840:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775443.260218:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775443.260219:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775443.260220:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775443.260221:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775443.260223:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775775443.260225:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775443.260227:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775443.260229:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775443.260231:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775443.260233:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710339840, portal 10 00000100:00000001:0.0:1775775443.260235:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775443.260236:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684359552 : -123823025192064 : ffff8f6234918380) 00000100:00000040:0.0:1775775443.260240:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918380 x1862022710339840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.260245:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775443.260246:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775443.260248:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775775443.260249:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710339840, offset 0 00000400:00000200:0.0:1775775443.260252:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775443.260254:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775443.260257:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775443.260259:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775443.260261:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775443.260262:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775443.260264:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775443.260266:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453207:453207:256:4294967295] 192.168.201.115@tcp LPNI seq info [453132:453132:8:4294967295] 00000400:00000200:0.0:1775775443.260269:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775443.260274:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775443.260277:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775443.260279:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cb00. 00000800:00000200:0.0:1775775443.260281:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775443.260283:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775775443.260286:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775443.260288:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1775775443.260290:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775443.260291:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cb00. 00000400:00000200:0.0:1775775443.260293:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775443.260296:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775443.260298:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775775443.260299:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775775443.260300:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775443.260302:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918380 x1862022710339840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.260308:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775443.260309:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918380 x1862022710339840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775468 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.260313:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775443.260314:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775443.260316:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775443.260319:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775443.260321:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775443.260322:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4306998284 00000100:00000001:0.0:1775775443.260324:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307002118 00000100:00000001:0.0:1775775443.260325:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4306998284 00000100:00000001:0.0:1775775443.260327:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307002118 00000100:00000001:0.0:1775775443.260328:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775443.260330:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775443.260331:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775443.260334:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775443/real 1775775443] req@ffff8f6234918380 x1862022710339840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775443.260340:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775443.260341:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775443.260343:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775443.260345:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775443.260347:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775443.260349:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775443.260352:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918380 x1862022710339840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775443.260361:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918380 x1862022710339840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.260368:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775443.260370:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775443.260371:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775443.260373:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775443.260377:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918380 x1862022710339840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.260384:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775443.260388:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918380 x1862022710339840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.260396:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775443.260399:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775443.260400:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775443.260402:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775443.260404:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775443.260409:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775443.260411:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775443.260414:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775443.260415:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775443.260418:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918380 x1862022710339840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775443.260423:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710339840:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775443.260427:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775443.260430:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918e00 x1862022710339968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.260437:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775443.260438:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775443.260441:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775443.260443:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775443.260445:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775443.260448:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710339968:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775443.260452:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775443.260454:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775443.260456:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775443.260458:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775443.260462:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775443.260464:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775443.260468:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775443.260470:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775443.260474:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775443.260477:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710339968, portal 10 00000100:00000001:0.0:1775775443.260479:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775443.260480:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684362240 : -123823025189376 : ffff8f6234918e00) 00000100:00000040:0.0:1775775443.260486:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918e00 x1862022710339968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.260493:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775443.260496:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775443.260499:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775775443.260502:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710339968, offset 0 00000400:00000200:0.0:1775775443.260506:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775443.260510:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775443.260514:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775443.260517:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775443.260519:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775443.260522:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775443.260524:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775443.260528:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453208:453208:256:4294967295] 192.168.201.115@tcp LPNI seq info [453133:453133:8:4294967295] 00000400:00000200:0.0:1775775443.260532:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775443.260539:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775443.260544:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775443.260548:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cb00. 00000800:00000200:0.0:1775775443.260552:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775443.260555:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775775443.260559:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775443.260562:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1775775443.260566:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775443.260568:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cb00. 00000400:00000200:0.0:1775775443.260571:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775443.260574:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775443.260577:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775775443.260579:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775775443.260582:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775443.260586:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918e00 x1862022710339968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.260595:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775443.260597:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918e00 x1862022710339968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775468 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.260604:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775443.260605:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775443.260608:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775443.260610:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775443.260613:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775443.260614:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4306998284 00000100:00000001:0.0:1775775443.260616:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307002118 00000100:00000001:0.0:1775775443.260618:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4306998284 00000100:00000001:0.0:1775775443.260619:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307002118 00000100:00000001:0.0:1775775443.260620:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775443.260622:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775443.260623:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775443.260625:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775443/real 1775775443] req@ffff8f6234918e00 x1862022710339968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775443.260631:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775443.260632:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775443.260633:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775443.260635:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775443.260636:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775443.260637:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775443.260639:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918e00 x1862022710339968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775443.260644:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918e00 x1862022710339968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.260648:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775443.260649:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775443.260650:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775443.260651:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775443.260653:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918e00 x1862022710339968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.260657:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775443.260659:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918e00 x1862022710339968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.260664:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775443.260666:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775443.260667:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775443.260668:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775443.260669:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775443.260670:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775443.260671:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775443.260673:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775443.260675:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775443.260677:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918e00 x1862022710339968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775443.260682:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710339968:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775443.260684:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775443.260685:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775443.260687:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918380 x1862022710339840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.260691:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775443.260692:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775443.260694:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775775443.260695:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775443.260696:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775443.260697:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775443.260698:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775443.260699:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:0.0:1775775443.260701:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775443.260702:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775443.260703:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918380. 00000100:00000001:0.0:1775775443.260705:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775443.260706:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775443.260707:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775443.260709:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918e00 x1862022710339968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775468 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775443.260713:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775443.260714:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775443.260715:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775443.260716:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775443.260717:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775443.260717:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775443.260718:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775443.260719:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4f400. 02000000:00000001:0.0:1775775443.260720:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775443.260721:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775443.260722:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918e00. 00000100:00000001:0.0:1775775443.260723:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775443.260724:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775443.260725:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775443.260727:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775443.260728:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775443.260729:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775443.260730:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775443.387205:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775443.387209:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775443.387214:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775443.387214:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775443.387217:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775443.387218:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775444.411269:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775444.411276:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775444.411281:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775444.411282:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775444.411285:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775444.411286:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775445.435233:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775445.435238:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775445.435243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775445.435243:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775445.435246:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775445.435247:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775445.435304:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775445.435307:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775445.435310:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775446.459237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775446.459242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775446.459247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775446.459248:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775446.459251:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775446.459252:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775447.163225:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307002118, at 4307002176 00000100:00000010:0.0:1775775447.163245:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00010000:00000001:3.0:1775775447.483248:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775447.483254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775447.483258:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775447.483261:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775447.483263:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775447.483266:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775447.483268:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775447.483270:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775447.483271:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775447.483273:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775447.483276:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775447.483276:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775447.483277:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775447.483278:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775447.483279:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775447.483280:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775447.483281:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775447.483282:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775447.483301:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775447.483302:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775447.483304:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775448.379253:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775448.379264:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775448.379268:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775448.379275:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775448.379281:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775448.379290:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12330 00000400:00000200:2.0:1775775448.379299:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775448.379306:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775448.379310:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775448.379314:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775448.379318:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775448.379322:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775448.379325:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775448.379329:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775448.379331:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775448.379334:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775448.379337:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775448.379340:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775448.379343:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775448.379347:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775448.379349:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775448.379351:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775448.379353:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775448.379356:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775448.379359:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775448.379362:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775448.379365:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775448.379367:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775448.379370:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775448.379373:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775448.379375:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775448.379384:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775775448.379387:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775448.379391:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775448.379393:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775448.379396:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775448.379398:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775448.379402:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775448.379410:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775775448.379417:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775448.379426:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710340096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775448.379437:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775448.379442:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775448.379447:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710340096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775448.379471:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775448.379474:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775448.379478:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775448.379480:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775448.379481:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775448.379484:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775448.379487:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12330 00000400:00000200:2.0:1775775448.379491:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775448.379494:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775448.379496:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775448.379499:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775448.379502:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775448.379505:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775448.379509:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775448.379512:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775448.379514:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775448.379516:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775448.379518:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775448.379520:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775448.379522:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775448.379526:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775448.379528:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775448.379531:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775448.379533:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775448.379535:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000001:0.0:1775775448.379539:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1775775448.379540:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775448.379546:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775448.379548:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1775775448.379549:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775775448.379555:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1775775448.379557:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775448.379560:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775448.379563:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775448.379567:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000040:0.0:1775775448.379571:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233448700 x1862022710340096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1775775448.379572:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775448.379578:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775448.379587:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775775448.379595:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775448.379600:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775448.379601:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775448.379603:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775448.379605:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775448.379609:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775448.379613:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000010:2.0:1775775448.379615:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:0.0:1775775448.379617:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775448.379619:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775448.379625:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:2.0:1775775448.379631:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710340224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1775775448.379633:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775448.379644:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775448.379661:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710340096:192.168.201.115@tcp:250:kworker.0 00000100:00000001:2.0:1775775448.379662:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775448.379669:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775448.379681:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710340224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775448.379683:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775448.379693:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1775775448.379696:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775448.379697:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775448.379701:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1775775448.379703:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1775775448.379709:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775448.379711:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775448.379713:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000010:0.0:1775775448.379716:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7800. 00000100:00000001:2.0:1775775448.379716:0:181953:0:(import.c:484:import_select_connection()) Process entered 02000000:00000001:0.0:1775775448.379724:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775448.379725:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12330 00000400:00000200:2.0:1775775448.379732:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775448.379736:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775448.379741:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1775775448.379743:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:2.0:1775775448.379747:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000010:0.0:1775775448.379752:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000100:00000001:2.0:1775775448.379753:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775448.379759:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000010:0.0:1775775448.379766:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000001:2.0:1775775448.379783:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775448.379786:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775448.379787:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775448.379789:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775448.379791:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775448.379793:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775448.379796:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775448.379804:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775448.379806:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775448.379808:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000200:0.0:1775775448.379811:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710340096, portal 25 00000100:00000001:2.0:1775775448.379812:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:0.0:1775775448.379820:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000040:2.0:1775775448.379820:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00000001:0.0:1775775448.379825:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662536960 : -123823047014656 : ffff8f6233448700) 00000100:00080000:2.0:1775775448.379828:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775448.379834:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775448.379837:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:0.0:1775775448.379840:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233448700 x1862022710340096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775473 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775775448.379842:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775448.379854:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775448.379861:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775448.379865:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775448.379875:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775448.379878:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1775775448.379879:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775448.379885:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000100:00000040:0.0:1775775448.379890:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000020:00000040:2.0:1775775448.379892:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000010:0.0:1775775448.379896:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 02000000:00000001:2.0:1775775448.379897:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775448.379903:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775775448.379906:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710340096, offset 0 00000100:00000001:2.0:1775775448.379906:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775448.379910:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775448.379914:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775448.379919:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000400:00000200:0.0:1775775448.379921:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:2.0:1775775448.379926:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775448.379934:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710340352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775775448.379939:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775448.379957:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:2.0:1775775448.379975:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1775775448.379977:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1775775448.379981:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775448.379989:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000040:2.0:1775775448.379990:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710340352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775775448.380002:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1775775448.380013:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775448.380014:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000040:2.0:1775775448.380020:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775775448.380029:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453209:453209:256:4294967295] 192.168.201.115@tcp LPNI seq info [453134:453134:8:4294967295] 00000100:00000040:2.0:1775775448.380031:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775448.380041:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775775448.380048:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:2.0:1775775448.380055:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775448.380060:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12341) 00000400:00000200:0.0:1775775448.380073:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775448.380083:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775448.380090:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cb00. 00000800:00000200:0.0:1775775448.380100:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775448.380107:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775775448.380130:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775448.380193:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775448.380197:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307007239 00000100:00000010:0.0:1775775448.380205:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775448.380214:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775448.380223:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307007239 00000100:00000010:0.0:1775775448.380228:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3780. 00000100:00000001:0.0:1775775448.380232:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775448.380236:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775448.380240:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775448.380245:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775448.380248:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775448.380258:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775448.380260:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1775775448.380263:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775448.380265:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775448.380266:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775448.380272:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449500 x1862022710340224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775448.380282:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775448.380284:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775448.380287:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775448.380290:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775448.380293:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775448.380296:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710340224:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775448.380301:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775448.380305:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775448.380307:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775448.380309:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775448.380313:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775448.380315:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775448.380320:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775448.380323:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888cb8. 00000400:00000010:0.0:1775775448.380328:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0550. 00000100:00000200:0.0:1775775448.380331:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710340224, portal 10 00000100:00000001:0.0:1775775448.380334:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775448.380335:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662540544 : -123823047011072 : ffff8f6233449500) 00000100:00000040:0.0:1775775448.380342:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449500 x1862022710340224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775473 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775448.380350:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775448.380353:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775448.380356:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc03b8. 00000100:00000200:0.0:1775775448.380358:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710340224, offset 0 00000400:00000200:0.0:1775775448.380363:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775448.380368:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775448.380372:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775448.380375:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775448.380378:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775448.380381:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775448.380383:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775448.380387:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453210:453210:255:4294967295] 192.168.201.115@tcp LPNI seq info [453135:453135:7:4294967295] 00000400:00000200:0.0:1775775448.380392:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775448.380399:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775448.380403:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775448.380407:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775775448.380411:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775448.380415:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775775448.380418:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775448.380423:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775448.380427:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775448.380429:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307003404 00000100:00000001:0.0:1775775448.380432:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307007239 00000100:00000001:0.0:1775775448.380435:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307003404 00000100:00000001:0.0:1775775448.380437:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307007239 00000100:00000001:0.0:1775775448.380440:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775448.380443:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775448.380445:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775448.380449:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449c00 x1862022710340352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775448.380459:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775448.380460:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775448.380463:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775448.380464:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775448.380466:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775448.380469:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710340352:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775448.380473:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775448.380475:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775448.380477:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775448.380479:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775448.380482:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775448.380484:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775448.380488:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775448.380491:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888738. 00000400:00000010:0.0:1775775448.380494:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775448.380497:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710340352, portal 10 00000100:00000001:0.0:1775775448.380499:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775448.380500:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662542336 : -123823047009280 : ffff8f6233449c00) 00000100:00000040:0.0:1775775448.380506:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449c00 x1862022710340352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775473 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775448.380513:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775448.380516:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775448.380519:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775775448.380522:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710340352, offset 0 00000400:00000200:0.0:1775775448.380526:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775448.380531:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775448.380535:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775448.380538:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775448.380541:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775448.380543:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775448.380546:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775448.380549:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453211:453211:254:4294967295] 192.168.201.115@tcp LPNI seq info [453136:453136:6:4294967295] 00000400:00000200:0.0:1775775448.380553:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775448.380560:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775448.380565:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775448.380569:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775775448.380573:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775448.380577:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775775448.380579:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775448.380583:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775448.380587:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775448.380589:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307003404 00000100:00000001:0.0:1775775448.380592:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307007239 00000100:00000001:0.0:1775775448.380595:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307003404 00000100:00000001:0.0:1775775448.380597:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307007239 00000100:00000001:0.0:1775775448.380600:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775448.380602:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775448.380604:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775448.380606:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775448.380610:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775448.380611:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1775775448.380614:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775448.380615:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775448.380617:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775448.380619:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775448.507289:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775448.507296:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775448.507302:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775448.507304:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775448.507307:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775448.507309:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775449.531224:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775449.531229:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775449.531233:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775449.531234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775449.531237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775449.531238:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775450.555264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775450.555270:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775450.555277:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775450.555278:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775450.555282:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775450.555284:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775450.555336:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775450.555339:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775450.555342:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775775451.451454:0:2401:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:3.0:1775775451.451499:0:2401:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1775775451.451511:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775451.451515:0:2401:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:3.0:1775775451.451521:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775451.451525:0:2401:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:3.0:1775775451.451530:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775775451.451534:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cb00. 00000400:00000200:3.0:1775775451.451539:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775775451.451544:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775775451.451551:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:3.0:1775775451.451554:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:3.0:1775775451.451559:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775775451.451568:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6233448700 x1862022710340096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775473 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775451.451611:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775775451.451616:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448700 x1862022710340096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775473 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775451.451624:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775451.451625:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775775451.451635:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775775451.451638:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:3.0:1775775451.451641:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775775451.451644:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775775451.451648:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc03b8 00000400:00000010:3.0:1775775451.451650:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc03b8. 00000100:00000001:3.0:1775775451.451652:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775775451.451657:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6233449500 x1862022710340224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775473 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775451.451665:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775775451.451668:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449500 x1862022710340224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775473 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775451.451675:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775451.451677:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775775451.451682:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775775451.451684:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:3.0:1775775451.451687:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775775451.451690:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775775451.451693:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:3.0:1775775451.451695:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:3.0:1775775451.451697:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775775451.451700:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6233449c00 x1862022710340352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775473 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775451.451708:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775775451.451710:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449c00 x1862022710340352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775473 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775451.451730:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775451.451732:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775451.451804:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775451.451808:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775451.451810:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775451.451817:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775448/real 1775775451] req@ffff8f6233449500 x1862022710340224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775451.451833:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775451.451835:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775451.451839:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888cb8. 00000400:00000200:0.0:1775775451.451842:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0550 00000400:00000010:0.0:1775775451.451844:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0550. 00000100:00000001:0.0:1775775451.451847:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775451.451851:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449500 x1862022710340224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775451.451860:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449500 x1862022710340224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775451.451868:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775451.451870:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775451.451872:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775451.451874:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775451.451886:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449500 x1862022710340224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775451.451892:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775451.451897:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449500 x1862022710340224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775451.451904:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775451.451907:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775451.451909:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775451.451911:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775451.451913:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775451.451920:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775451.451924:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775451.451928:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775451.451931:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775451.451935:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449500 x1862022710340224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775451.451943:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710340224:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775451.451948:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775451.451951:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775448/real 1775775451] req@ffff8f6233449c00 x1862022710340352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775451.451958:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775451.451959:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775451.451960:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888738. 00000400:00000200:0.0:1775775451.451963:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775451.451964:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775451.451966:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775451.451969:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449c00 x1862022710340352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775451.451977:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449c00 x1862022710340352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775451.451983:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775451.451984:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775451.451986:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775451.451987:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775451.451991:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449c00 x1862022710340352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775451.451997:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775451.452001:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449c00 x1862022710340352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775451.452007:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775451.452009:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775451.452011:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775451.452013:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775451.452014:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775451.452017:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775451.452019:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775451.452022:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775451.452025:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775451.452029:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449c00 x1862022710340352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775451.452036:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710340352:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775451.452040:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775451.452044:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775448/real 1775775451] req@ffff8f6233448700 x1862022710340096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775451.452052:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775451.452053:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775451.452055:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775451.452058:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775451.452060:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775451.452062:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775451.452065:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233448700 x1862022710340096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775451.452074:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233448700 x1862022710340096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775451.452080:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775451.452081:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775451.452083:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775451.452085:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775451.452088:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233448700 x1862022710340096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775451.452101:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775451.452105:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233448700 x1862022710340096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775451.452111:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775451.452114:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775451.452116:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775451.452117:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775451.452119:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775451.452120:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775451.452123:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775451.452128:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775451.452130:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775451.452856:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233448700 x1862022710340096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775451.452871:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710340096:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775451.452877:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775451.452880:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775451.452884:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449500 x1862022710340224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775451.452889:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775451.452892:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775451.452895:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775451.452899:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775451.452900:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775451.452902:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775451.452903:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775451.452905:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4ec00. 02000000:00000001:0.0:1775775451.452908:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775451.452910:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775451.452911:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449500. 00000100:00000001:0.0:1775775451.452914:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775451.452915:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775451.452916:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775451.452919:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449c00 x1862022710340352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775451.452923:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775451.452924:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775451.452925:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775451.452927:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775451.452928:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775451.452929:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775451.452930:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775451.452931:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4d400. 02000000:00000001:0.0:1775775451.452932:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775451.452933:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775451.452934:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449c00. 00000100:00000001:0.0:1775775451.452935:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775451.452936:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775451.452937:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775451.452939:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233448700 x1862022710340096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775473 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775451.452943:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775451.452944:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775451.452945:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775775451.452947:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775451.452948:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775451.452949:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775451.452952:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775451.452953:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4f400. 02000000:00000001:0.0:1775775451.452955:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775451.452956:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775451.452957:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233448700. 00000100:00000001:0.0:1775775451.452959:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775451.452961:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775451.452963:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775451.452972:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775451.452974:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775451.452977:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775451.452978:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775451.579216:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775451.579222:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775451.579223:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775451.579226:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775451.579229:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775451.579232:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775451.579234:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775451.579235:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775451.579238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775451.579239:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775451.579242:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775451.579249:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775452.283221:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307007239, at 4307007296 00000100:00000010:0.0:1775775452.283234:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00000800:00000200:2.0:1775775452.283234:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775775452.283244:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775775452.283250:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1775775452.283374:0:2405:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:3.0:1775775452.603223:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775452.603228:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775452.603233:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775452.603234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775452.603243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775452.603244:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775452.603263:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775452.603265:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775452.603267:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775453.499242:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775453.499250:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775453.499252:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775453.499255:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775453.499259:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775453.499267:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12336 00000400:00000200:2.0:1775775453.499278:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775453.499286:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775453.499290:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775453.499294:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775453.499298:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775453.499303:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775453.499308:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775453.499314:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775453.499318:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775453.499321:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775453.499326:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775453.499331:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775453.499335:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775453.499341:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775453.499347:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775453.499351:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775453.499353:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775453.499356:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775453.499361:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775453.499364:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775453.499367:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775453.499371:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775453.499374:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775453.499377:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775453.499379:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775453.499386:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775775453.499390:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775453.499394:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775453.499396:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775453.499399:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775453.499402:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775453.499406:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775453.499415:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775775453.499422:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775453.499432:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710340480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775453.499442:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775453.499447:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775453.499451:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710340480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775453.499479:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775453.499483:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775453.499487:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775453.499489:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775453.499490:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775453.499493:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775453.499497:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12336 00000400:00000200:2.0:1775775453.499503:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775453.499506:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775453.499509:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775453.499512:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775453.499515:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775453.499518:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775453.499520:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775453.499522:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775453.499524:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775453.499526:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775453.499529:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775453.499531:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775775453.499532:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1775775453.499534:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775775453.499538:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1775775453.499539:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775453.499542:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1775775453.499543:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775453.499547:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775453.499550:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:0.0:1775775453.499553:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344bb80 x1862022710340480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1775775453.499554:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775453.499561:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775453.499573:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775453.499574:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1775775453.499578:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775775453.499579:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775453.499584:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775775453.499585:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775453.499587:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775453.499589:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775453.499595:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775775453.499596:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1775775453.499601:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775775453.499602:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710340480:192.168.201.115@tcp:250:kworker.0 00000008:00000001:2.0:1775775453.499610:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775453.499611:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1775775453.499613:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775453.499616:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775453.499620:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775453.499621:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 02000000:00000001:0.0:1775775453.499623:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:2.0:1775775453.499625:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775453.499629:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775453.499630:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775453.499631:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000001:2.0:1775775453.499633:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775453.499636:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775453.499637:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775453.499640:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775453.499645:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000010:2.0:1775775453.499647:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000400:00000010:0.0:1775775453.499651:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000100:00000001:2.0:1775775453.499652:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775775453.499661:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00100000:2.0:1775775453.499661:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710340608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1775775453.499671:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710340480, portal 25 00000100:00000001:2.0:1775775453.499673:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775453.499690:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775453.499693:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775453.499697:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662550400 : -123823047001216 : ffff8f623344bb80) 00000100:00000040:2.0:1775775453.499697:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710340608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775775453.499709:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344bb80 x1862022710340480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775478 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775453.499712:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775453.499718:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775775453.499728:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1775775453.499728:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775453.499730:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775453.499732:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:0.0:1775775453.499736:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:2.0:1775775453.499738:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000400:00000010:0.0:1775775453.499741:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00080000:2.0:1775775453.499744:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12336 00000100:00000200:0.0:1775775453.499747:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710340480, offset 0 00000400:00000200:2.0:1775775453.499751:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775453.499759:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000400:00000200:0.0:1775775453.499760:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00080000:2.0:1775775453.499764:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775453.499768:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775453.499771:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775775453.499772:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:2.0:1775775453.499777:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775453.499780:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:0.0:1775775453.499783:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000020:00000001:2.0:1775775453.499784:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775453.499787:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775453.499790:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:0.0:1775775453.499792:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:2.0:1775775453.499793:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775453.499796:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775453.499798:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775453.499802:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775453.499804:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775453.499805:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775453.499807:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775453.499809:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775453.499813:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000400:00000200:0.0:1775775453.499818:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:2.0:1775775453.499819:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775453.499823:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775453.499826:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775453.499829:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775453.499831:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1775775453.499833:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:0.0:1775775453.499837:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000008:00100000:2.0:1775775453.499837:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775453.499843:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775453.499845:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453212:453212:256:4294967295] 192.168.201.115@tcp LPNI seq info [453137:453137:8:4294967295] 00000100:00000001:2.0:1775775453.499847:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775453.499854:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000010:2.0:1775775453.499855:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775775453.499859:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775453.499863:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775453.499867:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775453.499871:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:2.0:1775775453.499872:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:0.0:1775775453.499880:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 02000000:00000001:2.0:1775775453.499882:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775453.499885:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775775453.499887:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 02000000:00000010:2.0:1775775453.499891:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775775453.499896:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775775453.499897:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775453.499903:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000100:00100000:2.0:1775775453.499904:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710340736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.1:1775775453.499916:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:2.0:1775775453.499917:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775453.499921:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775775453.499924:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000100:00000040:2.0:1775775453.499930:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710340736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1775775453.499937:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775453.499941:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000100:00000001:2.0:1775775453.499947:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775453.499950:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:2.0:1775775453.499951:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775453.499958:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:0.0:1775775453.499959:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000040:2.0:1775775453.499964:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775775453.499968:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000100:00000040:2.0:1775775453.499969:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000010:0.0:1775775453.499973:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000040:2.0:1775775453.499977:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12346) 00000100:00000001:0.0:1775775453.499980:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775453.499987:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344bb80 x1862022710340480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775478 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.499999:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775453.500003:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344bb80 x1862022710340480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775478 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500011:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775453.500013:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775453.500019:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775453.500023:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775453.500026:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307012358 00000100:00000010:0.0:1775775453.500031:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775453.500038:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775453.500044:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307012358 00000100:00000010:0.0:1775775453.500048:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3f60. 00000100:00000001:0.0:1775775453.500051:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775453.500054:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775453.500057:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775453.500059:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775453.500064:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775453/real 1775775453] req@ffff8f623344bb80 x1862022710340480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775453.500074:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775453.500076:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775453.500078:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775453.500081:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775453.500082:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775453.500086:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775453.500089:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344bb80 x1862022710340480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775453.500098:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344bb80 x1862022710340480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500104:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775453.500106:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775453.500108:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775453.500111:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775453.500115:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344bb80 x1862022710340480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500122:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775453.500127:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344bb80 x1862022710340480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500179:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775453.500182:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775453.500184:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775453.500185:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775453.500188:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775453.500190:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775453.500194:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775453.500198:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775453.500201:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775453.500206:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344bb80 x1862022710340480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775453.500215:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710340480:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775453.500220:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775453.500222:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775453.500225:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344bb80 x1862022710340480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500232:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775453.500235:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775453.500238:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775453.500241:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775453.500243:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775453.500245:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775453.500248:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775453.500250:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4f000. 02000000:00000001:0.0:1775775453.500254:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775453.500256:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775453.500258:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344bb80. 00000100:00000001:0.0:1775775453.500262:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775453.500263:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775453.500265:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775453.500273:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775453.500274:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775453.500276:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775453.500278:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775453.500280:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775453.500284:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233448700 x1862022710340608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500292:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775453.500294:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775453.500296:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775453.500298:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775453.500300:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775453.500303:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710340608:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775453.500307:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775453.500309:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775453.500311:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775453.500313:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775453.500317:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775453.500319:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775453.500324:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775453.500328:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775453.500332:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775453.500335:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710340608, portal 10 00000100:00000001:0.0:1775775453.500338:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775453.500339:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662536960 : -123823047014656 : ffff8f6233448700) 00000100:00000040:0.0:1775775453.500346:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233448700 x1862022710340608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775478 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500353:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775453.500355:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775453.500358:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775453.500361:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710340608, offset 0 00000400:00000200:0.0:1775775453.500365:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775453.500369:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775453.500374:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775453.500377:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775453.500380:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775453.500383:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775453.500385:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775453.500389:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453213:453213:256:4294967295] 192.168.201.115@tcp LPNI seq info [453138:453138:8:4294967295] 00000400:00000200:0.0:1775775453.500393:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775453.500400:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775453.500405:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775453.500408:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775453.500413:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775453.500416:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775775453.500421:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775453.500424:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775775453.500428:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775453.500431:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775453.500434:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775453.500438:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775453.500441:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775453.500443:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775453.500445:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775453.500449:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233448700 x1862022710340608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775478 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500457:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775453.500460:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448700 x1862022710340608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775478 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500468:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775453.500470:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775453.500473:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775453.500478:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775453.500481:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775453.500483:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307008524 00000100:00000001:0.0:1775775453.500487:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307012358 00000100:00000001:0.0:1775775453.500490:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307008524 00000100:00000001:0.0:1775775453.500492:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307012358 00000100:00000001:0.0:1775775453.500495:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775453.500497:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775453.500500:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775453.500504:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775453/real 1775775453] req@ffff8f6233448700 x1862022710340608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775453.500513:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775453.500515:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775453.500516:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775453.500519:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775453.500520:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775453.500523:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775453.500526:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233448700 x1862022710340608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775453.500535:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233448700 x1862022710340608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500541:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775453.500543:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775453.500544:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775453.500546:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775453.500549:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233448700 x1862022710340608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500557:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775453.500561:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233448700 x1862022710340608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500568:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775453.500571:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775453.500573:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775453.500574:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775453.500576:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775453.500582:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775453.500584:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775453.500588:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775453.500591:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775453.500595:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233448700 x1862022710340608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775453.500602:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710340608:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775453.500606:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775453.500609:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449c00 x1862022710340736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500616:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775453.500617:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775453.500619:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775453.500621:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775453.500623:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775453.500626:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710340736:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775453.500630:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775453.500632:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775453.500634:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775453.500636:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775453.500640:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775453.500642:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775453.500646:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775453.500648:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775453.500651:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775453.500654:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710340736, portal 10 00000100:00000001:0.0:1775775453.500656:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775453.500658:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662542336 : -123823047009280 : ffff8f6233449c00) 00000100:00000040:0.0:1775775453.500663:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449c00 x1862022710340736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775478 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500671:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775453.500673:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775453.500676:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775453.500692:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710340736, offset 0 00000400:00000200:0.0:1775775453.500697:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775453.500702:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775453.500706:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775453.500708:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775453.500711:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775453.500714:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775453.500716:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775453.500719:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453214:453214:256:4294967295] 192.168.201.115@tcp LPNI seq info [453139:453139:8:4294967295] 00000400:00000200:0.0:1775775453.500723:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775453.500729:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775453.500735:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775453.500738:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775453.500742:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775453.500745:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775775453.500749:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775453.500752:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775775453.500756:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775453.500758:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775453.500761:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775453.500764:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775453.500767:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775453.500768:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775453.500771:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775453.500774:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449c00 x1862022710340736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775478 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500788:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775453.500791:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449c00 x1862022710340736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775478 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500797:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775453.500799:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775453.500802:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775453.500805:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775453.500808:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775453.500810:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307008524 00000100:00000001:0.0:1775775453.500812:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307012358 00000100:00000001:0.0:1775775453.500814:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307008524 00000100:00000001:0.0:1775775453.500817:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307012358 00000100:00000001:0.0:1775775453.500819:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775453.500821:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775453.500823:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775453.500827:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775453/real 1775775453] req@ffff8f6233449c00 x1862022710340736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775453.500835:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775453.500836:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775453.500838:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775453.500840:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775453.500842:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775453.500844:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775453.500846:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449c00 x1862022710340736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775453.500854:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449c00 x1862022710340736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500860:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775453.500862:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775453.500864:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775453.500865:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775453.500868:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449c00 x1862022710340736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500875:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775453.500879:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449c00 x1862022710340736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500886:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775453.500888:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775453.500891:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775453.500892:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775453.500895:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775453.500897:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775453.500899:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775453.500902:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775453.500905:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775453.500909:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449c00 x1862022710340736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775453.500916:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710340736:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775453.500920:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775453.500921:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775453.500925:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233448700 x1862022710340608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500932:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775453.500934:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775453.500936:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775453.500939:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775453.500940:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775453.500942:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775453.500944:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775453.500946:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4d800. 02000000:00000001:0.0:1775775453.500949:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775453.500950:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775453.500951:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233448700. 00000100:00000001:0.0:1775775453.500954:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775453.500956:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775453.500958:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775453.500961:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449c00 x1862022710340736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775478 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775453.500968:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775453.500969:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775453.500970:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775453.500972:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775453.500974:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775453.500975:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775453.500977:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775453.500978:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4cc00. 02000000:00000001:0.0:1775775453.500981:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775453.500982:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775453.500984:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449c00. 00000100:00000001:0.0:1775775453.500986:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775453.500988:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775453.500991:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775453.500995:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775453.500996:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775453.500998:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775453.501000:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775453.627213:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775453.627218:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775453.627223:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775453.627224:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775453.627227:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775453.627228:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775454.651269:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775454.651278:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775454.651284:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775454.651285:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775454.651289:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775454.651290:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775455.675270:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775455.675275:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775455.675280:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775455.675281:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775455.675284:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775455.675285:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775455.675362:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775455.675367:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775455.675372:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775456.699292:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775456.699299:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775456.699304:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775456.699305:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775456.699309:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775456.699311:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775457.403225:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307012358, at 4307012416 00000100:00000010:0.0:1775775457.403241:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00010000:00000001:3.0:1775775457.723290:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775457.723297:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775457.723302:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775457.723306:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775457.723308:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775457.723311:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775457.723314:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775457.723316:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775457.723317:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775457.723319:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775457.723322:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775457.723323:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775457.723324:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775457.723325:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775457.723326:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775457.723327:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775457.723328:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775457.723329:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775457.723355:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775457.723357:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775457.723359:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775458.619281:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775458.619294:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775458.619298:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775458.619303:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775458.619314:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775458.619325:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12341 00000400:00000200:3.0:1775775458.619336:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775458.619342:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775458.619346:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775458.619353:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775458.619358:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775458.619361:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775458.619365:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775458.619369:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775458.619371:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775458.619374:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775458.619377:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775458.619380:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775458.619382:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775458.619386:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775458.619388:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775458.619390:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775458.619392:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775458.619395:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775458.619398:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775458.619401:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775458.619404:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775458.619407:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775458.619410:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775458.619412:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775458.619414:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775458.619421:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775775458.619424:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775458.619427:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775458.619428:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775458.619431:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775458.619433:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775458.619436:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775458.619443:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000100:00000001:3.0:1775775458.619448:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775458.619455:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710340864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775458.619464:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775458.619467:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775458.619471:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710340864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775458.619489:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775458.619492:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775458.619494:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775458.619495:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775458.619496:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775458.619499:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775458.619500:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12341 00000400:00000200:3.0:1775775458.619503:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775458.619505:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775458.619507:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775458.619509:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775458.619510:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775458.619512:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775458.619513:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775458.619516:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775458.619517:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775458.619518:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775458.619519:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775458.619521:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775458.619523:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775458.619525:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775458.619527:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775458.619528:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775458.619530:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775458.619532:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000001:0.0:1775775458.619535:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1775775458.619536:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775458.619541:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775458.619542:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1775775458.619545:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775775458.619547:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1775775458.619552:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1775775458.619557:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71180 x1862022710340864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775775458.619557:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775458.619561:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775458.619567:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1775775458.619572:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000008:00000001:3.0:1775775458.619573:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775458.619575:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775458.619577:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775458.619580:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:3.0:1775775458.619582:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 02000000:00000001:0.0:1775775458.619584:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775458.619599:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775775458.619599:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775458.619605:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00100000:0.0:1775775458.619606:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710340864:192.168.201.115@tcp:250:kworker.0 02000000:00000001:3.0:1775775458.619608:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775458.619613:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1775775458.619615:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775775458.619616:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775458.619619:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775458.619620:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775458.619624:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775458.619625:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 02000000:00000001:0.0:1775775458.619627:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775775458.619630:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775458.619635:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 00000100:00100000:3.0:1775775458.619638:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710340992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1775775458.619639:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775458.619651:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775775458.619655:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:0.0:1775775458.619657:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000100:00000001:3.0:1775775458.619659:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775458.619665:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710340992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1775775458.619666:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775458.619675:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710340864, portal 25 00000100:00000001:3.0:1775775458.619678:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775458.619679:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775458.619683:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849008512 : -123823860543104 : ffff8f6202c71180) 00000100:00080000:3.0:1775775458.619684:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775458.619690:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000040:0.0:1775775458.619693:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71180 x1862022710340864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775483 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775458.619693:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775458.619696:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775458.619701:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775458.619707:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12341 00000100:00000001:0.0:1775775458.619710:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1775775458.619714:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:0.0:1775775458.619718:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000200:3.0:1775775458.619722:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000400:00000010:0.0:1775775458.619723:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00080000:3.0:1775775458.619726:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000200:0.0:1775775458.619728:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710340864, offset 0 00000100:00000040:3.0:1775775458.619731:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775458.619734:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775458.619736:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775458.619738:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:0.0:1775775458.619742:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000020:00000001:3.0:1775775458.619742:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775458.619744:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775458.619746:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775458.619748:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775458.619749:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775458.619751:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:0.0:1775775458.619754:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775775458.619755:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775458.619758:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775458.619760:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775458.619762:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775458.619763:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:0.0:1775775458.619766:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00080000:3.0:1775775458.619768:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775458.619772:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775458.619773:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1775775458.619775:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1775775458.619777:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775458.619779:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775458.619781:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:0.0:1775775458.619782:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000008:00100000:3.0:1775775458.619784:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775458.619787:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775458.619789:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1775775458.619789:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775458.619793:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000400:00000200:0.0:1775775458.619794:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775458.619799:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453215:453215:256:4294967295] 192.168.201.115@tcp LPNI seq info [453140:453140:8:4294967295] 00000020:00000040:3.0:1775775458.619805:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:0.0:1775775458.619806:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775775458.619811:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775458.619813:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775458.619816:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775458.619818:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775458.619820:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775458.619822:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 02000000:00000010:3.0:1775775458.619824:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775775458.619829:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775775458.619833:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00100000:3.0:1775775458.619836:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710341120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:0.0:1775775458.619842:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000100:00000001:3.0:1775775458.619843:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775458.619846:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775458.619849:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710341120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1775775458.619856:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775775458.619858:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775458.619862:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.0:1775775458.619863:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000100:00000040:3.0:1775775458.619867:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775458.619870:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775458.619875:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:0.1:1775775458.619875:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775775458.619882:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12351) 00000800:00000200:0.1:1775775458.619882:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1775775458.619889:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775458.619892:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775458.619899:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775458.619905:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775458.619910:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775458.619913:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775458.619917:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775458.619922:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71180 x1862022710340864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775483 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.619931:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775458.619935:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71180 x1862022710340864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775483 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.619943:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775458.619944:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775458.619950:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775458.619954:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775458.619957:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307017478 00000100:00000010:0.0:1775775458.619963:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775458.619969:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775458.619977:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307017478 00000100:00000010:0.0:1775775458.619980:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3420. 00000100:00000001:0.0:1775775458.619984:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775458.619987:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775458.619990:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775458.619993:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775458.619997:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775458/real 1775775458] req@ffff8f6202c71180 x1862022710340864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775458.620007:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775458.620008:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775458.620010:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775458.620013:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775458.620014:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775458.620017:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775458.620020:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71180 x1862022710340864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775458.620030:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71180 x1862022710340864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.620036:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775458.620038:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775458.620040:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775458.620042:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775458.620046:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71180 x1862022710340864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.620053:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775458.620058:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71180 x1862022710340864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.620065:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775458.620068:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775458.620070:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775458.620071:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775458.620073:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775458.620075:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775458.620078:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775458.620083:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775458.620085:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775458.620089:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71180 x1862022710340864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775458.620098:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710340864:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775458.620103:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775458.620105:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775458.620108:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71180 x1862022710340864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.620116:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775458.620118:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775458.620121:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775458.620124:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775458.620126:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775458.620127:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775458.620129:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775458.620131:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36000. 02000000:00000001:0.0:1775775458.620165:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775458.620167:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775458.620169:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71180. 00000100:00000001:0.0:1775775458.620172:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775458.620174:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775458.620176:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775458.620182:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775458.620184:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775458.620186:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775458.620188:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775458.620190:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775458.620203:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72a00 x1862022710340992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.620211:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775458.620212:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775458.620215:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775458.620217:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775458.620219:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775458.620222:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710340992:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775458.620225:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775458.620227:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775458.620229:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775458.620231:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775458.620234:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775458.620236:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775458.620240:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775458.620243:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775458.620247:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775458.620250:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710340992, portal 10 00000100:00000001:0.0:1775775458.620252:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775458.620253:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849014784 : -123823860536832 : ffff8f6202c72a00) 00000100:00000040:0.0:1775775458.620259:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72a00 x1862022710340992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775483 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.620266:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775458.620268:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775458.620270:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775458.620272:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710340992, offset 0 00000400:00000200:0.0:1775775458.620276:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775458.620280:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775458.620284:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775458.620287:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775458.620289:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775458.620292:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775458.620294:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775458.620298:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453216:453216:256:4294967295] 192.168.201.115@tcp LPNI seq info [453141:453141:8:4294967295] 00000400:00000200:0.0:1775775458.620302:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775458.620308:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775458.620314:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775458.620317:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775458.620330:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775458.620333:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775775458.620338:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775458.620341:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1775775458.620345:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775458.620347:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775458.620350:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775458.620354:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775458.620357:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775458.620359:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775458.620362:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775458.620365:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72a00 x1862022710340992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775483 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.620373:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775458.620376:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72a00 x1862022710340992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775483 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.620383:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775458.620385:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775458.620388:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775458.620392:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775458.620396:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775458.620398:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307013644 00000100:00000001:0.0:1775775458.620401:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307017478 00000100:00000001:0.0:1775775458.620404:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307013644 00000100:00000001:0.0:1775775458.620406:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307017478 00000100:00000001:0.0:1775775458.620408:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775458.620411:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775458.620413:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775458.620416:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775458/real 1775775458] req@ffff8f6202c72a00 x1862022710340992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775458.620425:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775458.620426:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775458.620428:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775458.620430:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775458.620432:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775458.620434:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775458.620437:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72a00 x1862022710340992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775458.620446:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72a00 x1862022710340992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.620452:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775458.620454:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775458.620456:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775458.620457:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775458.620461:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72a00 x1862022710340992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.620468:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775458.620471:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72a00 x1862022710340992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.620479:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775458.620481:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775458.620483:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775458.620484:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775458.620486:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775458.620491:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775458.620494:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775458.620497:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775458.620500:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775458.620504:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72a00 x1862022710340992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775458.620513:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710340992:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775458.620517:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775458.620521:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72300 x1862022710341120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.620528:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775458.620530:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775458.620532:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775458.620534:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775458.620536:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775458.620539:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710341120:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775458.620543:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775458.620545:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775458.620546:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775458.620548:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775458.620552:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775458.620554:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775458.620557:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775458.620560:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775458.620563:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775458.620566:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710341120, portal 10 00000100:00000001:0.0:1775775458.620568:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775458.620569:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849012992 : -123823860538624 : ffff8f6202c72300) 00000100:00000040:0.0:1775775458.620575:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72300 x1862022710341120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775483 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.620582:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775458.620584:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775458.620586:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775458.620603:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710341120, offset 0 00000400:00000200:0.0:1775775458.620608:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775458.620612:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775458.620616:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775458.620619:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775458.620622:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775458.620624:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775458.620627:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775458.620630:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453217:453217:256:4294967295] 192.168.201.115@tcp LPNI seq info [453142:453142:8:4294967295] 00000400:00000200:0.0:1775775458.620634:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775458.620641:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775458.620647:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775458.620650:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775458.620655:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775458.620658:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775775458.620663:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775458.620665:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1775775458.620670:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775458.620672:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775458.620675:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775458.620679:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775458.620682:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775458.620684:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775458.620687:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775458.620691:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72300 x1862022710341120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775483 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.620700:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775458.620703:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72300 x1862022710341120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775483 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.620711:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775458.620713:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775458.620716:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775458.620719:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775458.620723:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775458.620725:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307013644 00000100:00000001:0.0:1775775458.620728:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307017478 00000100:00000001:0.0:1775775458.620730:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307013644 00000100:00000001:0.0:1775775458.620733:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307017478 00000100:00000001:0.0:1775775458.620735:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775458.620738:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775458.620740:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775458.620744:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775458/real 1775775458] req@ffff8f6202c72300 x1862022710341120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775458.620753:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775458.620754:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775458.620756:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775458.620759:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775458.620761:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775458.620763:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775458.620767:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72300 x1862022710341120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775458.620776:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72300 x1862022710341120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.620784:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775458.620785:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775458.620787:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775458.620789:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775458.620792:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72300 x1862022710341120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.620800:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775458.620804:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72300 x1862022710341120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.620812:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775458.620815:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775458.620817:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775458.620818:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775458.620820:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775458.620822:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775458.620825:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775458.620829:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775458.620831:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775458.620835:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72300 x1862022710341120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775458.620844:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710341120:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775458.620847:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775458.620849:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775458.620853:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72a00 x1862022710340992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.620860:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775458.620862:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775458.620864:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775458.620867:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775458.620868:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775458.620870:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775458.620872:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775458.620874:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36400. 02000000:00000001:0.0:1775775458.620876:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775458.620878:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775458.620879:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72a00. 00000100:00000001:0.0:1775775458.620882:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775458.620883:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775458.620885:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775458.620888:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72300 x1862022710341120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775483 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775458.620901:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775458.620902:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775458.620904:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775458.620906:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775458.620907:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775458.620908:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775458.620910:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775458.620911:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34c00. 02000000:00000001:0.0:1775775458.620912:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775458.620913:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775458.620914:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72300. 00000100:00000001:0.0:1775775458.620915:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775458.620916:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775458.620918:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775458.620921:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775458.620922:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775458.620924:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775458.620926:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775458.747219:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775458.747226:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775458.747231:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775458.747232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775458.747235:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775458.747236:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775459.771272:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775459.771279:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775459.771284:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775459.771285:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775459.771288:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775459.771289:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775460.795240:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775460.795249:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775460.795257:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775460.795259:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775460.795264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775460.795265:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775460.795309:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775460.795314:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775460.795317:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775461.819238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775461.819244:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775461.819245:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775461.819248:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775461.819252:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775461.819254:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775461.819256:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775461.819257:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775461.819260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775461.819261:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775461.819264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775461.819265:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775462.523264:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307017478, at 4307017536 00000100:00000010:0.0:1775775462.523277:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00010000:00000001:3.0:1775775462.843226:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775462.843231:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775462.843235:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775462.843236:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775462.843239:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775462.843240:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775462.843260:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775462.843262:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775462.843266:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775463.739315:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775463.739325:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775463.739328:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775463.739332:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775463.739337:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775463.739344:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12346 00000400:00000200:3.0:1775775463.739354:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775463.739360:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775463.739364:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775463.739367:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775463.739371:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775463.739374:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775463.739376:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775463.739380:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775463.739383:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775463.739385:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775463.739388:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775463.739391:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775463.739393:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775463.739396:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775463.739398:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775463.739400:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775463.739403:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775463.739406:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775463.739409:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775463.739412:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775463.739416:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775463.739419:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775463.739422:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775463.739424:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775463.739426:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775463.739436:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73b80. 00000020:00000040:3.0:1775775463.739441:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775463.739445:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775463.739447:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775463.739450:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775463.739453:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775463.739457:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775463.739465:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000100:00000001:3.0:1775775463.739472:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775463.739481:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73b80 x1862022710341248/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775463.739493:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775463.739497:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775463.739518:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c52008f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73b80 x1862022710341248/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775463.739551:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775463.739555:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775463.739559:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775463.739560:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775463.739562:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775463.739565:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775463.739567:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12346 00000400:00000200:3.0:1775775463.739572:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775463.739576:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775463.739579:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775463.739582:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775463.739585:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775463.739588:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775463.739590:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775463.739593:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775463.739594:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775463.739596:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775463.739599:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775463.739601:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775775463.739602:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1775775463.739604:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775775463.739608:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775775463.739609:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775463.739612:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775775463.739613:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775463.739617:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775463.739620:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:0.0:1775775463.739624:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73b80 x1862022710341248/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775775463.739624:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775463.739631:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775463.739637:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775463.739643:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1775775463.739643:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775775463.739646:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775463.739648:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775463.739649:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775463.739652:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775463.739653:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775463.739657:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775463.739658:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00100000:0.0:1775775463.739663:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710341248:192.168.201.115@tcp:250:kworker.0 00000008:00100000:3.0:1775775463.739664:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1775775463.739669:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00000001:3.0:1775775463.739670:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775463.739673:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1775775463.739674:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775463.739676:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775463.739679:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:3.0:1775775463.739681:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 02000000:00000010:0.0:1775775463.739684:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 00000020:00000040:3.0:1775775463.739686:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775463.739688:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775463.739691:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775463.739694:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775463.739696:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775775463.739698:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:0.0:1775775463.739701:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 02000000:00000001:3.0:1775775463.739701:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775463.739705:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775775463.739708:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 02000000:00000010:3.0:1775775463.739712:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000200:0.0:1775775463.739715:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710341248, portal 25 00000100:00000001:3.0:1775775463.739717:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775463.739718:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775463.739721:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849019264 : -123823860532352 : ffff8f6202c73b80) 00000100:00100000:3.0:1775775463.739725:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710341376/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1775775463.739728:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73b80 x1862022710341248/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775488 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.739754:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775775463.739755:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:0.0:1775775463.739759:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775775463.739761:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775775463.739763:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775463.739766:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710341248, offset 0 00000100:00000040:3.0:1775775463.739770:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710341376/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775775463.739775:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775463.739784:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775775463.739787:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775463.739792:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00080000:3.0:1775775463.739792:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775463.739799:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775463.739801:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1775775463.739802:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:3.0:1775775463.739804:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1775775463.739809:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:3.0:1775775463.739810:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775463.739816:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12346 00000400:00000200:0.0:1775775463.739818:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775463.739825:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:3.0:1775775463.739826:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775463.739832:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453218:453218:256:4294967295] 192.168.201.115@tcp LPNI seq info [453143:453143:8:4294967295] 00000400:00000200:3.0:1775775463.739836:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775463.739842:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1775775463.739845:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775775463.739849:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775463.739852:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775463.739856:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775463.739860:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775463.739866:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:0.0:1775775463.739867:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000020:00000040:3.0:1775775463.739869:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775463.739873:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:0.0:1775775463.739877:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000020:00000040:3.0:1775775463.739877:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775463.739882:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000010:0.0:1775775463.739883:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000020:00000001:3.0:1775775463.739885:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775463.739891:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000800:00000200:0.0:1775775463.739892:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775775463.739896:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.0:1775775463.739898:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000100:00000040:3.0:1775775463.739903:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775463.739907:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000200:0.1:1775775463.739908:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000040:3.0:1775775463.739913:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000800:00000200:0.1:1775775463.739914:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000100:00080000:3.0:1775775463.739922:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000800:00000100:0.0:1775775463.739923:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775463.739927:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000100:00000001:3.0:1775775463.739928:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775463.739932:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1775775463.739933:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:3.0:1775775463.739936:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775775463.739940:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 02000000:00000001:3.0:1775775463.739941:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775463.739945:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:0.0:1775775463.739946:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000008:00100000:3.0:1775775463.739947:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000010:0.0:1775775463.739951:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000008:00000001:3.0:1775775463.739951:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775463.739956:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775463.739959:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:3.0:1775775463.739961:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000100:00000200:0.0:1775775463.739966:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73b80 x1862022710341248/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775488 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775775463.739966:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775463.739971:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775463.739974:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775463.739978:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775463.739981:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775463.739984:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775463.739987:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000010:3.0:1775775463.739989:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000040:0.0:1775775463.739993:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73b80 x1862022710341248/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775488 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740003:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775463.740005:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775775463.740006:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775463.740013:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775463.740015:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710341504/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775775463.740023:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775463.740029:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307022598 00000100:00000001:3.0:1775775463.740031:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775463.740035:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775463.740038:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000040:3.0:1775775463.740041:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710341504/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740047:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775775463.740054:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775463.740057:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307022598 00000100:00000040:3.0:1775775463.740057:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000010:0.0:1775775463.740063:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3420. 00000100:00000040:3.0:1775775463.740065:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1775775463.740067:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000040:3.0:1775775463.740070:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775775463.740071:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775463.740076:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775463.740079:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000040:3.0:1775775463.740079:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775463.740087:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12356) 00000100:00100000:0.0:1775775463.740088:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775463/real 1775775463] req@ffff8f6202c73b80 x1862022710341248/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775463.740101:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775463.740103:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775463.740106:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775463.740111:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775463.740113:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775463.740116:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775463.740120:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73b80 x1862022710341248/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775463.740130:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73b80 x1862022710341248/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740175:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775463.740176:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775463.740178:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775463.740180:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775463.740183:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73b80 x1862022710341248/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740188:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775463.740191:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73b80 x1862022710341248/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740196:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775463.740198:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775463.740199:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775463.740200:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775463.740201:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775463.740203:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775463.740204:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775463.740207:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775463.740209:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775463.740211:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73b80 x1862022710341248/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775463.740217:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710341248:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775463.740222:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775463.740224:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775463.740226:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73b80 x1862022710341248/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740231:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775463.740233:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775463.740235:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775463.740237:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775463.740238:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775463.740239:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775463.740240:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775463.740242:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34400. 02000000:00000001:0.0:1775775463.740245:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775463.740247:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775463.740248:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73b80. 00000100:00000001:0.0:1775775463.740251:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775463.740252:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775463.740253:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775463.740263:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775463.740265:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775463.740267:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775463.740268:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775463.740269:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775463.740271:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72300 x1862022710341376/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740276:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775463.740278:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775463.740279:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775463.740280:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775463.740282:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775463.740284:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710341376:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775463.740286:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775463.740288:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775463.740289:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775463.740291:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775463.740293:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775463.740295:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775463.740298:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775463.740300:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775463.740303:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775463.740305:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710341376, portal 10 00000100:00000001:0.0:1775775463.740307:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775463.740308:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849012992 : -123823860538624 : ffff8f6202c72300) 00000100:00000040:0.0:1775775463.740312:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72300 x1862022710341376/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775488 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740317:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775463.740319:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775463.740320:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775463.740322:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710341376, offset 0 00000400:00000200:0.0:1775775463.740324:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775463.740327:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775463.740329:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775463.740331:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775463.740333:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775463.740335:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775463.740337:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775463.740339:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453219:453219:256:4294967295] 192.168.201.115@tcp LPNI seq info [453144:453144:8:4294967295] 00000400:00000200:0.0:1775775463.740342:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775463.740347:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775463.740350:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775463.740352:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775463.740355:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775463.740357:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775775463.740361:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775463.740363:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1775775463.740366:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775463.740367:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775463.740369:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775463.740372:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775463.740374:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775463.740375:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775463.740377:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775463.740379:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72300 x1862022710341376/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775488 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740384:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775463.740386:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72300 x1862022710341376/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775488 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740390:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775463.740391:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775463.740393:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775463.740397:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775463.740399:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775463.740400:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307018764 00000100:00000001:0.0:1775775463.740402:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307022598 00000100:00000001:0.0:1775775463.740403:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307018764 00000100:00000001:0.0:1775775463.740405:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307022598 00000100:00000001:0.0:1775775463.740406:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775463.740408:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775463.740409:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775463.740417:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775463/real 1775775463] req@ffff8f6202c72300 x1862022710341376/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775463.740423:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775463.740426:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775463.740428:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775463.740430:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775463.740431:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775463.740432:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775463.740434:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72300 x1862022710341376/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775463.740440:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72300 x1862022710341376/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740444:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775463.740445:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775463.740446:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775463.740447:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775463.740449:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72300 x1862022710341376/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740453:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775463.740455:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72300 x1862022710341376/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740460:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775463.740461:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775463.740462:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775463.740463:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775463.740464:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775463.740468:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775463.740470:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775463.740472:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775463.740474:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775463.740478:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72300 x1862022710341376/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775463.740486:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710341376:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775463.740490:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775463.740493:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72a00 x1862022710341504/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740518:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775463.740520:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775463.740522:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775463.740524:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775463.740526:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775463.740529:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710341504:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775463.740533:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775463.740535:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775463.740537:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775463.740539:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775463.740543:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775463.740545:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775463.740549:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775463.740552:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775463.740555:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775463.740558:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710341504, portal 10 00000100:00000001:0.0:1775775463.740559:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775463.740560:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849014784 : -123823860536832 : ffff8f6202c72a00) 00000100:00000040:0.0:1775775463.740565:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72a00 x1862022710341504/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775488 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740570:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775463.740572:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775463.740573:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775463.740575:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710341504, offset 0 00000400:00000200:0.0:1775775463.740577:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775463.740580:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775463.740583:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775463.740585:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775463.740586:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775463.740588:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775463.740590:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775463.740592:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453220:453220:256:4294967295] 192.168.201.115@tcp LPNI seq info [453145:453145:8:4294967295] 00000400:00000200:0.0:1775775463.740595:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775463.740599:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775463.740602:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775463.740605:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775463.740607:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775463.740609:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775775463.740613:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775463.740616:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1775775463.740619:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775463.740622:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775463.740624:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775463.740628:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775463.740631:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775463.740632:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775463.740635:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775463.740638:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72a00 x1862022710341504/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775488 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740647:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775463.740650:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72a00 x1862022710341504/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775488 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740656:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775463.740658:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775463.740661:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775463.740664:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775463.740667:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775463.740670:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307018764 00000100:00000001:0.0:1775775463.740672:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307022598 00000100:00000001:0.0:1775775463.740674:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307018764 00000100:00000001:0.0:1775775463.740676:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307022598 00000100:00000001:0.0:1775775463.740678:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775463.740681:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775463.740683:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775463.740688:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775463/real 1775775463] req@ffff8f6202c72a00 x1862022710341504/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775463.740696:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775463.740698:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775463.740700:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775463.740702:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775463.740704:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775463.740706:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775463.740709:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72a00 x1862022710341504/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775463.740717:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72a00 x1862022710341504/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740723:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775463.740724:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775463.740725:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775463.740726:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775463.740728:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72a00 x1862022710341504/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740733:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775463.740735:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72a00 x1862022710341504/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740739:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775463.740741:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775463.740742:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775463.740742:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775463.740744:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775463.740745:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775463.740746:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775463.740749:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775463.740750:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775463.740753:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72a00 x1862022710341504/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775463.740757:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710341504:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775463.740760:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775463.740761:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775463.740763:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72300 x1862022710341376/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740767:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775463.740768:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775463.740769:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775463.740770:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775463.740771:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775463.740772:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775463.740774:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775463.740775:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34800. 02000000:00000001:0.0:1775775463.740777:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775463.740778:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775463.740779:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72300. 00000100:00000001:0.0:1775775463.740781:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775463.740782:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775463.740783:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775463.740784:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72a00 x1862022710341504/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775488 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775463.740788:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775463.740789:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775463.740790:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775463.740791:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775463.740792:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775463.740793:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775463.740794:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775463.740795:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37000. 02000000:00000001:0.0:1775775463.740797:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775463.740798:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775463.740798:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72a00. 00000100:00000001:0.0:1775775463.740800:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775463.740801:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775463.740802:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775463.740804:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775463.740805:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775463.740807:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775463.740808:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775463.867259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775463.867266:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775463.867273:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775463.867274:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775463.867278:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775463.867280:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775464.891256:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775464.891264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775464.891273:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775464.891274:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775464.891278:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775464.891280:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775465.915252:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775465.915258:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775465.915264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775465.915265:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775465.915268:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775465.915269:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775465.915334:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775465.915339:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775465.915343:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775466.939239:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775466.939246:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775466.939252:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775466.939253:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775466.939257:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775466.939259:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775467.643239:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307022598, at 4307022656 00000100:00000010:0.0:1775775467.643258:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00010000:00000001:3.0:1775775467.963282:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775467.963287:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775467.963293:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775467.963297:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775467.963299:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775467.963302:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775467.963305:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775467.963307:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775467.963309:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775467.963311:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775467.963316:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775467.963317:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775467.963319:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775467.963320:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775467.963322:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775467.963323:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775467.963325:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775467.963327:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775467.963357:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775467.963360:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775467.963362:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775468.859235:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775468.859246:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775468.859250:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775468.859254:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775468.859257:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775468.859263:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12351 00000400:00000200:3.0:1775775468.859271:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775468.859279:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775468.859282:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775468.859285:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775468.859288:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775468.859290:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775468.859292:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775468.859295:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775468.859296:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775468.859299:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775468.859301:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775468.859303:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775468.859305:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775468.859306:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775468.859308:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775468.859309:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775468.859311:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775468.859313:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775468.859315:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775468.859317:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775468.859319:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775468.859328:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775468.859331:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775468.859332:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775468.859334:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775468.859341:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775775468.859344:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775468.859346:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775468.859348:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775468.859350:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775468.859352:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775468.859356:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775468.859362:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:3.0:1775775468.859368:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775468.859375:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710341632/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775468.859383:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775468.859386:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775468.859390:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710341632/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775468.859407:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775468.859421:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775468.859424:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775468.859426:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775468.859427:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775468.859429:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775468.859431:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12351 00000400:00000200:3.0:1775775468.859434:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775468.859436:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775468.859438:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775468.859441:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775468.859442:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775468.859445:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775468.859446:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775468.859448:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775468.859449:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775468.859451:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775468.859452:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:0.0:1775775468.859454:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1775775468.859454:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775468.859456:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775775468.859458:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775775468.859459:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775468.859462:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775775468.859462:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775468.859466:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775468.859467:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:0.0:1775775468.859471:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71180 x1862022710341632/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775775468.859471:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775468.859476:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775468.859482:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775468.859487:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775775468.859489:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775468.859491:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775468.859492:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775468.859494:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775468.859495:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1775775468.859496:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775775468.859498:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775468.859500:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775775468.859500:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775468.859506:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775468.859507:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710341632:192.168.201.115@tcp:250:kworker.0 00000100:00000001:3.0:1775775468.859508:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775468.859514:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000010:3.0:1775775468.859515:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 02000000:00000001:0.0:1775775468.859518:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:3.0:1775775468.859518:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775468.859520:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775468.859521:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775468.859523:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775468.859524:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775775468.859525:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775468.859528:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775468.859529:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775468.859530:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775468.859533:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775468.859534:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775775468.859538:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775468.859539:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00100000:3.0:1775775468.859544:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710341760/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1775775468.859545:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775468.859555:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000001:3.0:1775775468.859562:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:0.0:1775775468.859564:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710341632, portal 25 00000100:00000001:3.0:1775775468.859565:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775468.859569:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775468.859573:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849008512 : -123823860543104 : ffff8f6202c71180) 00000100:00000040:3.0:1775775468.859576:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710341760/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775775468.859583:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71180 x1862022710341632/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775493 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.859601:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775775468.859601:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775468.859606:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00080000:3.0:1775775468.859607:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:0.0:1775775468.859609:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775468.859614:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710341632, offset 0 00000100:00080000:3.0:1775775468.859615:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775468.859618:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775468.859620:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1775775468.859623:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775775468.859626:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775468.859631:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12351 00000400:00000200:0.0:1775775468.859632:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775468.859640:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775468.859641:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:3.0:1775775468.859648:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000400:00000200:0.0:1775775468.859649:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:3.0:1775775468.859653:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1775775468.859657:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000040:3.0:1775775468.859659:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775468.859662:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775775468.859664:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000040:3.0:1775775468.859668:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:0.0:1775775468.859670:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:3.0:1775775468.859672:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775468.859676:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:0.0:1775775468.859677:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453221:453221:256:4294967295] 192.168.201.115@tcp LPNI seq info [453146:453146:8:4294967295] 00000020:00000040:3.0:1775775468.859677:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775468.859681:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775468.859684:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000400:00000200:0.0:1775775468.859685:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000001:3.0:1775775468.859688:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775468.859691:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775468.859695:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775468.859699:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775775468.859702:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000040:3.0:1775775468.859706:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:0.0:1775775468.859711:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775775468.859712:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775468.859716:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000800:00000010:0.0:1775775468.859717:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000100:00080000:3.0:1775775468.859722:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000800:00000200:0.0:1775775468.859727:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775775468.859728:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775775468.859732:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 02000000:00000001:3.0:1775775468.859733:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775468.859738:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:0.1:1775775468.859738:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:3.0:1775775468.859744:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775775468.859745:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000100:00000001:3.0:1775775468.859750:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000100:0.0:1775775468.859751:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775468.859755:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000008:00100000:3.0:1775775468.859755:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775468.859759:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775468.859761:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1775775468.859761:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775468.859764:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000400:00000200:0.0:1775775468.859768:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000020:00000040:3.0:1775775468.859769:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775468.859775:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775468.859777:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775468.859778:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000100:00000001:3.0:1775775468.859778:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775468.859780:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775468.859783:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775775468.859784:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 02000000:00000010:3.0:1775775468.859787:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000001:0.0:1775775468.859789:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1775775468.859790:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775775468.859797:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71180 x1862022710341632/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775493 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775775468.859798:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710341888/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775468.859816:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775468.859820:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775468.859825:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710341888/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.859826:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775468.859833:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71180 x1862022710341632/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775493 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775468.859838:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775468.859843:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775468.859851:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1775775468.859855:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775468.859856:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775775468.859860:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1775775468.859861:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775468.859867:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12361) 00000100:00000001:0.0:1775775468.859869:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775468.859877:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775468.859881:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307027718 00000100:00000010:0.0:1775775468.859887:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775468.859893:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775468.859898:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307027718 00000100:00000010:0.0:1775775468.859901:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3f60. 00000100:00000001:0.0:1775775468.859904:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775468.859907:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775468.859910:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775468.859912:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775468.859917:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775468/real 1775775468] req@ffff8f6202c71180 x1862022710341632/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775468.859926:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775468.859928:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775468.859930:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775468.859933:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775468.859935:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775468.859937:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775468.859940:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71180 x1862022710341632/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775468.859950:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71180 x1862022710341632/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.859957:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775468.859959:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775468.859961:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775468.859963:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775468.859966:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71180 x1862022710341632/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.859973:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775468.859977:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71180 x1862022710341632/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.859984:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775468.859987:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775468.859989:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775468.859990:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775468.859993:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775468.859995:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775468.859998:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775468.860003:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775468.860005:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775468.860010:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71180 x1862022710341632/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775468.860019:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710341632:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775468.860024:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775468.860026:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775468.860029:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71180 x1862022710341632/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.860036:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775468.860039:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775468.860042:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775468.860045:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775468.860047:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775468.860048:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775468.860050:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775468.860052:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35400. 02000000:00000001:0.0:1775775468.860055:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775468.860056:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775468.860057:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71180. 00000100:00000001:0.0:1775775468.860059:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775468.860060:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775468.860062:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775468.860067:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775468.860068:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775468.860070:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775468.860071:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775468.860072:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775468.860075:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70a80 x1862022710341760/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.860080:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775468.860081:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775468.860082:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775468.860084:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775468.860085:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775468.860086:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710341760:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775468.860089:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775468.860090:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775468.860091:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775468.860093:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775468.860095:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775468.860096:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775468.860099:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775468.860101:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775468.860103:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775468.860105:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710341760, portal 10 00000100:00000001:0.0:1775775468.860107:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775468.860108:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849006720 : -123823860544896 : ffff8f6202c70a80) 00000100:00000040:0.0:1775775468.860112:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70a80 x1862022710341760/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775493 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.860117:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775468.860118:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775468.860119:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775468.860121:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710341760, offset 0 00000400:00000200:0.0:1775775468.860124:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775468.860126:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775468.860129:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775468.860131:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775468.860133:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775468.860167:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775468.860169:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775468.860171:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453222:453222:256:4294967295] 192.168.201.115@tcp LPNI seq info [453147:453147:8:4294967295] 00000400:00000200:0.0:1775775468.860174:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775468.860179:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775468.860182:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775468.860185:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775468.860188:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775468.860190:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775775468.860193:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775468.860195:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1775775468.860197:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775468.860199:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775468.860201:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775468.860203:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775468.860206:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775468.860207:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775468.860208:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775468.860211:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70a80 x1862022710341760/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775493 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.860216:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775468.860218:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70a80 x1862022710341760/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775493 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.860227:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775468.860228:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775468.860230:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775468.860233:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775468.860235:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775468.860237:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307023884 00000100:00000001:0.0:1775775468.860239:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307027718 00000100:00000001:0.0:1775775468.860240:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307023884 00000100:00000001:0.0:1775775468.860242:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307027718 00000100:00000001:0.0:1775775468.860243:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775468.860245:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775468.860246:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775468.860248:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775468/real 1775775468] req@ffff8f6202c70a80 x1862022710341760/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775468.860253:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775468.860254:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775468.860255:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775468.860257:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775468.860258:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775468.860259:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775468.860261:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70a80 x1862022710341760/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775468.860266:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70a80 x1862022710341760/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.860270:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775468.860271:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775468.860272:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775468.860273:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775468.860275:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70a80 x1862022710341760/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.860279:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775468.860282:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70a80 x1862022710341760/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.860286:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775468.860288:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775468.860289:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775468.860289:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775468.860290:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775468.860294:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775468.860295:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775468.860297:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775468.860299:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775468.860301:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70a80 x1862022710341760/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775468.860305:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710341760:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775468.860308:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775468.860310:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70700 x1862022710341888/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.860314:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775468.860315:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775468.860317:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775468.860318:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775468.860319:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775468.860321:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710341888:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775468.860322:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775468.860323:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775468.860325:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775468.860326:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775468.860328:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775468.860330:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775468.860332:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775468.860333:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775468.860335:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775468.860337:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710341888, portal 10 00000100:00000001:0.0:1775775468.860338:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775468.860339:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849005824 : -123823860545792 : ffff8f6202c70700) 00000100:00000040:0.0:1775775468.860343:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70700 x1862022710341888/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775493 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.860347:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775468.860349:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775468.860350:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775468.860352:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710341888, offset 0 00000400:00000200:0.0:1775775468.860354:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775468.860356:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775468.860358:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775468.860360:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775468.860361:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775468.860363:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775468.860364:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775468.860367:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453223:453223:256:4294967295] 192.168.201.115@tcp LPNI seq info [453148:453148:8:4294967295] 00000400:00000200:0.0:1775775468.860369:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775468.860374:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775468.860376:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775468.860378:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775468.860380:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775468.860382:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775775468.860385:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775468.860387:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1775775468.860390:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775468.860392:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775468.860393:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775468.860395:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775468.860397:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775468.860398:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775468.860400:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775468.860402:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70700 x1862022710341888/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775493 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.860407:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775468.860423:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70700 x1862022710341888/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775493 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.860431:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775468.860432:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775468.860436:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775468.860439:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775468.860442:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775468.860444:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307023884 00000100:00000001:0.0:1775775468.860447:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307027718 00000100:00000001:0.0:1775775468.860449:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307023884 00000100:00000001:0.0:1775775468.860451:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307027718 00000100:00000001:0.0:1775775468.860454:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775468.860456:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775468.860458:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775468.860462:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775468/real 1775775468] req@ffff8f6202c70700 x1862022710341888/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775468.860471:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775468.860472:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775468.860474:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775468.860476:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775468.860478:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775468.860480:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775468.860482:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70700 x1862022710341888/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775468.860488:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70700 x1862022710341888/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.860492:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775468.860493:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775468.860494:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775468.860495:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775468.860497:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70700 x1862022710341888/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.860501:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775468.860504:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70700 x1862022710341888/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.860508:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775468.860510:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775468.860511:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775468.860512:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775468.860513:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775468.860514:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775468.860515:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775468.860517:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775468.860519:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775468.860521:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70700 x1862022710341888/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775468.860526:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710341888:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775468.860528:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775468.860530:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775468.860531:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70a80 x1862022710341760/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.860536:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775468.860537:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775468.860538:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775468.860539:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775468.860540:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775468.860541:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775468.860543:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775468.860544:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37400. 02000000:00000001:0.0:1775775468.860546:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775468.860547:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775468.860548:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70a80. 00000100:00000001:0.0:1775775468.860549:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775468.860550:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775468.860551:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775468.860553:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70700 x1862022710341888/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775493 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775468.860557:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775468.860558:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775468.860558:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775468.860560:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775468.860560:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775468.860561:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775468.860562:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775468.860563:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35800. 02000000:00000001:0.0:1775775468.860564:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775468.860565:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775468.860565:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70700. 00000100:00000001:0.0:1775775468.860566:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775468.860567:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775468.860568:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775468.860570:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775468.860571:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775468.860572:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775468.860574:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775468.987238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775468.987242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775468.987247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775468.987248:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775468.987253:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775468.987255:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775470.011243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775470.011250:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775470.011256:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775470.011257:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775470.011261:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775470.011262:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775471.035244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775471.035248:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775471.035252:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775471.035254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775471.035256:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775471.035257:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775471.035322:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775471.035326:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775471.035329:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775472.059243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775472.059250:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775472.059253:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775472.059257:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775472.059262:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775472.059264:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775472.059267:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775472.059269:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775472.059273:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775472.059275:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775472.059279:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775472.059281:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775472.763230:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307027718, at 4307027776 00000100:00000010:0.0:1775775472.763249:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00010000:00000001:3.0:1775775473.083242:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775473.083258:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775473.083265:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775473.083266:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775473.083270:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775473.083271:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775473.083295:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775473.083299:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775473.083302:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775473.979275:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775473.979284:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775473.979291:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775473.979299:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775473.979310:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775473.979318:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12356 00000400:00000200:0.0:1775775473.979345:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775473.979351:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775473.979354:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775473.979357:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775473.979361:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775473.979365:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775473.979367:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775473.979370:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775473.979372:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775473.979374:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775473.979377:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775473.979378:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775473.979379:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775473.979381:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775473.979383:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775473.979384:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775473.979385:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775473.979387:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775473.979390:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775473.979392:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775473.979395:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775473.979397:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775473.979399:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775473.979401:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775473.979403:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775473.979411:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062d80. 00000020:00000040:0.0:1775775473.979414:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775473.979417:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775473.979418:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775473.979420:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775473.979423:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775473.979426:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775473.979433:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000001:0.0:1775775473.979439:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775473.979446:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062d80 x1862022710342016/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775473.979454:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775473.979458:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775473.979461:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009965f80b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062d80 x1862022710342016/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775473.979485:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775473.979488:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775473.979490:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775473.979492:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775473.979493:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775473.979495:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775473.979497:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12356 00000400:00000200:0.0:1775775473.979500:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775473.979502:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775473.979504:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775473.979512:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775473.979515:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775473.979517:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775473.979518:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775473.979520:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775473.979524:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775473.979526:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775473.979534:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775473.979539:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775473.979541:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775473.979551:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775473.979560:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775473.979574:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775473.979582:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775473.979592:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775473.979603:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775473.979607:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775473.979613:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775473.979646:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775473.979655:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775473.979659:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775473.979663:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775473.979671:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775473.979679:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775473.979686:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062a00. 00000020:00000040:0.0:1775775473.979692:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775473.979708:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775473.979713:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775473.979721:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775473.979726:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775473.979730:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775473.979738:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6000. 00000100:00000001:0.0:1775775473.979743:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775473.979750:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062a00 x1862022710342144/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775473.979765:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775473.979767:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775473.979770:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009753f9f6] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062a00 x1862022710342144/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775473.979790:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775473.979797:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775473.979802:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775473.979807:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775473.979810:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775473.979816:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775473.979825:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12356 00000400:00000200:0.0:1775775473.979866:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775473.979876:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775473.979879:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775473.979893:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775473.979902:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1775775473.979906:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:0.0:1775775473.979911:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:1.1:1775775473.979914:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000100:00000001:0.0:1775775473.979916:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775473.979924:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775473.979927:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775473.979934:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775473.979940:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775473.979948:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775473.979951:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775473.979963:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775473.979971:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775473.979980:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775473.979987:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775473.979991:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775473.980001:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775473.980007:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775473.980010:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775473.980015:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775473.980022:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775473.980027:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775473.980034:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775473.980042:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775473.980048:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775473.980058:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061500. 00000020:00000040:0.0:1775775473.980064:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775473.980078:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775473.980081:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775473.980089:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775473.980093:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775473.980097:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775473.980104:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4400. 00000100:00000001:0.0:1775775473.980113:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775473.980126:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061500 x1862022710342272/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775473.980191:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775473.980197:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775473.980212:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb8d992] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061500 x1862022710342272/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775473.980230:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775473.980236:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775473.980255:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775473.980269:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775473.980278:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775473.980287:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12366) 00000800:00000200:1.1:1775775473.980654:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775473.980657:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775775473.980970:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775473.980973:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00010000:00000001:3.0:1775775474.107269:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775474.107275:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775474.107280:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775474.107281:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775474.107284:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775474.107285:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775475.131234:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775475.131241:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775475.131246:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775475.131247:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775475.131251:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775475.131251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775476.155274:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775476.155281:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775476.155299:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775476.155300:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775476.155304:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775476.155305:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775476.155380:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775476.155384:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775476.155389:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775477.179221:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775477.179230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775477.179238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775477.179240:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775477.179244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775477.179246:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775477.883230:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307032838, at 4307032896 00000100:00000010:0.0:1775775477.883287:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2300. 00010000:00000001:3.0:1775775478.203272:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775478.203277:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775478.203281:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775478.203284:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775478.203286:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775478.203288:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775478.203290:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775478.203292:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775478.203294:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775478.203295:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775478.203298:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775478.203299:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775478.203300:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775478.203301:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775478.203302:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775478.203303:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775478.203304:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775478.203305:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775478.203324:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775478.203326:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775478.203329:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775479.099273:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775479.099283:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775479.099288:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775479.099294:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775479.099299:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775479.099306:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12361 00000400:00000200:3.0:1775775479.099314:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775479.099323:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775479.099326:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775479.099328:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775479.099331:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775479.099332:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775479.099334:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775479.099337:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775479.099339:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775479.099341:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775479.099343:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775479.099344:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775479.099346:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775479.099347:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775479.099349:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775479.099350:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775479.099351:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775479.099354:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775479.099356:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775479.099357:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775479.099360:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775479.099362:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775479.099364:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775479.099366:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775479.099367:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775479.099374:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775775479.099376:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775479.099379:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775479.099380:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775479.099382:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775479.099384:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775479.099388:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775479.099395:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000001:3.0:1775775479.099400:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775479.099407:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710342400/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775479.099415:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775479.099419:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775479.099422:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710342400/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775479.099436:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775479.099439:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775479.099441:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775479.099442:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775479.099443:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775479.099445:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775479.099447:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12361 00000400:00000200:3.0:1775775479.099450:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775479.099452:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775479.099455:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775479.099459:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775479.099461:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775479.099463:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775479.099464:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775479.099467:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775479.099471:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775479.099476:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775479.099479:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775479.099481:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775479.099482:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775479.099489:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775479.099493:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775479.099501:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775479.099507:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775479.099513:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775479.099522:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775479.099533:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775479.099539:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775479.099549:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775479.099552:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775479.099560:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775479.099567:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775479.099572:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775479.099577:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775479.099584:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775775479.099589:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775479.099598:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775479.099599:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775479.099600:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775479.099602:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775479.099608:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775479.099613:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:3.0:1775775479.099617:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775479.099630:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710342528/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775479.099639:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775479.099641:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775479.099649:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710342528/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775479.099670:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775479.099672:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775479.099682:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775479.099687:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775479.099694:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775479.099705:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775479.099711:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12361 00000400:00000200:3.0:1775775479.099721:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775479.099729:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775479.099738:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775479.099744:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775479.099752:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775479.099756:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775479.099758:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775479.099760:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775479.099765:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775479.099769:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775479.099772:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775479.099775:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775479.099781:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775479.099789:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775479.099796:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775479.099802:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775479.099810:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775479.099813:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775479.099816:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775479.099817:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775479.099821:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775479.099832:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775479.099838:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775479.099843:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775479.099849:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775479.099861:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775479.099866:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775479.099870:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775775479.099872:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775479.099881:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775479.099885:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775479.099892:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775479.099894:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775479.099896:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775479.099905:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000100:00000001:3.0:1775775479.099912:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775479.099918:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710342656/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775479.099927:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775479.099933:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775479.099941:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710342656/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1775775479.099977:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775479.099989:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000100:00000001:3.0:1775775479.099991:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775479.099997:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775479.100003:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775479.100010:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775479.100020:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775479.100026:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12371) 00000800:00000200:1.1:1775775479.100402:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775479.100404:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775775479.100610:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775479.100611:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00010000:00000001:3.0:1775775479.227247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775479.227254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775479.227259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775479.227261:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775479.227265:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775479.227266:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775480.251243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775480.251249:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775480.251253:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775480.251255:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775480.251257:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775480.251258:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775481.275292:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775481.275298:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775481.275305:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775481.275306:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775481.275310:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775481.275311:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775481.275369:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775481.275372:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775481.275374:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775482.299285:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775482.299296:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775482.299299:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775482.299304:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775482.299310:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775482.299313:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775482.299315:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775482.299318:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775482.299323:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775482.299324:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775482.299329:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775482.299331:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775483.003264:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307037958, at 4307038016 00000100:00000010:0.0:1775775483.003285:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2de0. 00010000:00000001:3.0:1775775483.323289:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775483.323295:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775483.323300:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775483.323301:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775483.323304:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775483.323305:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775483.323330:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775483.323332:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775483.323334:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775484.219275:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775484.219284:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775484.219288:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775484.219293:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775484.219297:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775484.219306:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12366 00000400:00000200:0.0:1775775484.219318:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775484.219325:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775484.219329:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775484.219332:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775484.219335:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775484.219338:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775484.219340:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775484.219343:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775484.219344:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775484.219346:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775484.219349:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775484.219350:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775484.219354:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775484.219356:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775484.219357:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775484.219359:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775484.219360:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775484.219363:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775484.219365:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775484.219367:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775484.219370:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775484.219372:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775484.219375:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775484.219376:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775484.219378:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775484.219389:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063b80. 00000020:00000040:0.0:1775775484.219392:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775484.219396:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775484.219397:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775484.219399:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775484.219402:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775484.219407:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775484.219415:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000001:0.0:1775775484.219423:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775484.219432:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063b80 x1862022710342784/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775484.219442:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775484.219447:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775484.219450:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ee9007b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063b80 x1862022710342784/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775484.219470:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775484.219473:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775484.219476:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775484.219477:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775484.219479:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775484.219481:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775484.219483:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12366 00000400:00000200:0.0:1775775484.219486:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775484.219488:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775484.219489:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775484.219491:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775484.219494:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775484.219495:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775484.219497:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775484.219498:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775484.219500:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775484.219501:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775484.219502:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775484.219510:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775484.219514:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775484.219516:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775484.219518:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775484.219519:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775484.219520:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775484.219522:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775484.219527:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775484.219530:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775484.219532:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775484.219538:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775484.219543:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775484.219547:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775484.219553:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775484.219556:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775484.219559:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775484.219562:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061f80. 00000020:00000040:0.0:1775775484.219566:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775484.219569:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775484.219572:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775484.219575:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775484.219576:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775484.219579:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775484.219582:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6000. 00000100:00000001:0.0:1775775484.219588:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775484.219593:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061f80 x1862022710342912/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775484.219606:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775484.219610:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775484.219615:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000029d3794] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061f80 x1862022710342912/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775484.219628:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775484.219632:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775484.219634:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775484.219638:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775484.219640:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775484.219645:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775484.219650:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12366 00000400:00000200:0.0:1775775484.219659:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775484.219665:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775484.219668:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775484.219674:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775484.219677:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775484.219679:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775484.219683:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775484.219685:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775484.219687:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775484.219690:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775484.219692:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775484.219694:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775484.219695:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775484.219699:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775484.219702:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775484.219706:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775484.219708:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775484.219710:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775484.219712:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775484.219713:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775484.219714:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775484.219716:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775484.219719:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775484.219724:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775484.219725:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775484.219727:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775484.219728:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775484.219731:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060700. 00000020:00000040:0.0:1775775484.219735:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775484.219737:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775484.219738:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775484.219739:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775484.219741:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775484.219745:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775484.219748:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7800. 00000100:00000001:0.0:1775775484.219752:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775484.219759:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060700 x1862022710343040/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775484.219769:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775484.219773:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775484.219785:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000225c8695] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060700 x1862022710343040/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775484.219793:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775484.219795:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775484.219803:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775484.219808:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775484.219815:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775484.219820:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12376) 00000800:00000200:1.1:1775775484.219845:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775484.219848:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775775484.220201:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775484.220203:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775775484.220409:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775484.220411:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00010000:00000001:3.0:1775775484.347237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775484.347244:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775484.347249:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775484.347250:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775484.347253:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775484.347254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775485.371276:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775485.371282:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775485.371289:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775485.371290:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775485.371294:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775485.371295:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775486.395222:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775486.395228:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775486.395234:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775486.395235:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775486.395238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775486.395239:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775486.395286:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775486.395290:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775486.395293:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775487.419273:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775487.419278:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775487.419283:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775487.419286:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775487.419289:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775487.419292:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775487.419295:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775487.419297:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775487.419299:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775487.419301:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775487.419304:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775487.419305:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775487.419305:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775487.419306:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775487.419307:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775487.419308:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775487.419309:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775487.419310:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775488.123263:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307043078, at 4307043136 00000100:00000010:0.0:1775775488.123274:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2660. 00010000:00000001:3.0:1775775488.443240:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775488.443247:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775488.443252:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775488.443254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775488.443257:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775488.443258:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775488.443285:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775488.443289:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775488.443292:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775489.339245:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775489.339258:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775489.339263:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775489.339270:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775489.339286:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775489.339298:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12371 00000400:00000200:0.0:1775775489.339314:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775489.339323:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775489.339328:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775489.339332:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775489.339336:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775489.339338:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775489.339341:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775489.339346:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775489.339348:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775489.339352:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775489.339356:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775489.339358:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775489.339361:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775489.339364:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775489.339366:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775489.339368:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775489.339370:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775489.339373:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775489.339377:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775489.339379:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775489.339383:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775489.339386:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775489.339390:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775489.339392:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775489.339395:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775489.339407:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061880. 00000020:00000040:0.0:1775775489.339411:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775489.339416:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775489.339418:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775489.339421:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775489.339425:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775489.339430:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775489.339440:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6400. 00000100:00000001:0.0:1775775489.339447:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775489.339467:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061880 x1862022710343168/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775489.339480:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775489.339486:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775489.339490:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005afd1f27] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061880 x1862022710343168/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775489.339515:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775489.339519:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775489.339523:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775489.339524:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775489.339526:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775489.339529:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775489.339531:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12371 00000400:00000200:0.0:1775775489.339535:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775489.339539:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775489.339542:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775489.339545:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775489.339548:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775489.339551:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775489.339554:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775489.339557:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775489.339558:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775489.339562:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775489.339564:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775489.339569:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775489.339574:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775489.339581:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775489.339588:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775489.339592:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775489.339597:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775489.339600:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775489.339605:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775489.339611:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775489.339615:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775489.339623:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775489.339629:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775489.339632:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775489.339637:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775489.339643:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775489.339646:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775489.339651:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061180. 00000020:00000040:0.0:1775775489.339657:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775489.339662:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775489.339664:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775489.339666:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775489.339669:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775489.339671:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775489.339677:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5800. 00000100:00000001:0.0:1775775489.339681:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775489.339688:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061180 x1862022710343296/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775489.339700:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775489.339703:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775489.339711:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e23859] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061180 x1862022710343296/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775489.339727:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775489.339733:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775489.339743:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775489.339746:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775489.339749:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775489.339760:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775489.339765:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12371 00000400:00000200:0.0:1775775489.339772:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775489.339779:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775489.339783:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775489.339787:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775489.339789:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775489.339794:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775489.339798:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775489.339801:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775489.339805:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775489.339809:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775489.339812:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775489.339816:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775489.339820:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775489.339825:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775489.339827:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775489.339829:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775489.339831:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775489.339836:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775489.339843:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775489.339848:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775489.339852:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775489.339857:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775489.339863:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775489.339869:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775489.339872:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775489.339878:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775489.339880:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775489.339884:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061c00. 00000020:00000040:0.0:1775775489.339889:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775489.339894:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775489.339897:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775489.339900:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775489.339902:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775489.339906:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775489.339911:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7000. 00000100:00000001:0.0:1775775489.339916:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775489.339924:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061c00 x1862022710343424/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1775775489.339945:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775489.339951:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000100:00000001:0.0:1775775489.339959:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775489.339963:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775489.339972:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076a44879] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061c00 x1862022710343424/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775489.339989:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775489.339994:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775489.340001:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775489.340006:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775489.340010:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775489.340018:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12382) 00000800:00000200:1.1:1775775489.340456:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775489.340459:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775775489.340739:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775489.340741:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00010000:00000001:3.0:1775775489.467279:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775489.467286:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775489.467292:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775489.467293:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775489.467297:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775489.467299:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775490.491300:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775490.491309:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775490.491316:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775490.491318:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775490.491322:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775490.491323:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775775491.195319:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:0.0:1775775491.195329:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:0.0:1775775491.195335:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1775775491.195575:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775775491.195597:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775775491.195604:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775775491.195607:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1775775491.515264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775491.515272:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775491.515278:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775491.515280:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775491.515284:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775491.515285:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775491.515356:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775491.515359:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775491.515363:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775492.539244:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775492.539266:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775492.539269:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775492.539274:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775492.539280:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775492.539283:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775492.539285:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775492.539294:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775492.539299:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775492.539301:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775492.539306:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775492.539307:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775493.243217:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307048198, at 4307048256 00000100:00000010:0.0:1775775493.243226:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2b40. 00010000:00000001:3.0:1775775493.563259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775493.563264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775493.563268:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775493.563269:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775493.563272:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775493.563273:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775493.563293:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775493.563295:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775493.563297:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775494.459296:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775494.459320:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775494.459323:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775494.459328:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775494.459334:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775494.459342:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12377 00000400:00000200:2.0:1775775494.459355:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775494.459364:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775494.459370:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775494.459378:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775494.459385:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775494.459392:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775494.459398:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775494.459405:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775494.459409:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775494.459421:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775494.459427:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775494.459431:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775494.459434:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775494.459439:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775494.459441:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775494.459443:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775494.459445:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775494.459447:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775494.459450:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775494.459452:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775494.459455:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775494.459457:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775494.459460:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775494.459462:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775494.459463:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775494.459471:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775775494.459474:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775494.459478:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775494.459480:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775494.459483:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775494.459486:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775494.459490:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775494.459497:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775775494.459504:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775494.459513:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710343552/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775494.459521:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775494.459526:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775494.459531:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710343552/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775494.459551:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775494.459554:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775494.459558:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775494.459560:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775494.459562:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775494.459565:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775494.459567:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12377 00000400:00000200:2.0:1775775494.459572:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775494.459575:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775494.459578:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775494.459581:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775494.459583:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775494.459585:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775494.459587:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775494.459590:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775494.459592:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775494.459593:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775494.459596:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775494.459598:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775494.459600:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775494.459604:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775494.459606:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775494.459608:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775494.459610:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775494.459613:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775494.459615:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775494.459617:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775494.459619:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775494.459626:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775494.459628:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775494.459630:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775494.459636:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775494.459639:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775494.459651:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775494.459655:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775775494.459657:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775494.459660:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775494.459661:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775494.459663:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775494.459665:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775494.459667:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775494.459670:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775775494.459674:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775494.459678:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710343680/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775494.459685:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775494.459688:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775494.459692:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710343680/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775494.459701:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775494.459704:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775494.459709:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775494.459710:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775494.459712:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775494.459715:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775494.459717:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12377 00000400:00000200:2.0:1775775494.459720:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775494.459723:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775494.459726:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775494.459729:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775494.459731:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775494.459734:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775494.459736:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775494.459738:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775494.459740:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775494.459742:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775494.459743:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775494.459746:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775494.459748:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775494.459751:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775494.459754:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775494.459755:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775494.459758:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775494.459760:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775494.459762:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775494.459765:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775494.459767:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775494.459769:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775494.459772:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775494.459774:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775494.459775:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775494.459778:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775494.459780:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775494.459783:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775775494.459785:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775494.459788:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775494.459790:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775494.459791:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775494.459793:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775494.459794:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775494.459798:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775775494.459801:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775494.459805:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710343808/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775494.459813:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775494.459815:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775494.459819:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710343808/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1775775494.459822:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775494.459825:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1775775494.459827:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775494.459830:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775494.459835:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775494.459838:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775494.459841:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775494.459846:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12387) 00000800:00000200:1.1:1775775494.460204:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775494.460206:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775775494.460397:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775494.460399:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00010000:00000001:3.0:1775775494.587266:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775494.587272:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775494.587276:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775494.587277:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775494.587280:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775494.587280:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775495.611234:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775495.611240:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775495.611246:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775495.611247:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775495.611251:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775495.611253:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775496.635270:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775496.635277:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775496.635282:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775496.635283:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775496.635288:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775496.635289:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775496.635361:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775496.635365:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775496.635369:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775497.659210:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775497.659216:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775497.659222:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775497.659226:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775497.659228:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775497.659231:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775497.659235:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775497.659237:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775497.659240:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775497.659243:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775497.659247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775497.659248:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775497.659250:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775497.659251:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775497.659253:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775497.659255:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775497.659256:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775497.659258:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775498.363259:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307053318, at 4307053376 00000100:00000010:0.0:1775775498.363267:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e20c0. 00010000:00000001:3.0:1775775498.683291:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775498.683296:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775498.683302:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775498.683303:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775498.683306:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775498.683307:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775498.683331:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775498.683334:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775498.683337:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775499.579297:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775499.579313:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775499.579315:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775499.579321:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775499.579329:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775499.579337:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12382 00000400:00000200:0.0:1775775499.579352:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775499.579359:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775499.579371:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775499.579378:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775499.579381:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775499.579383:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775499.579386:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775499.579389:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775499.579391:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775499.579394:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775499.579396:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775499.579398:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775499.579399:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775499.579402:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775499.579403:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775499.579404:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775499.579406:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775499.579408:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775499.579410:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775499.579412:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775499.579414:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775499.579416:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775499.579418:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775499.579419:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775499.579421:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775499.579429:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060e00. 00000020:00000040:0.0:1775775499.579432:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775499.579435:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775499.579437:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775499.579439:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775499.579442:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775499.579445:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775499.579452:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7c00. 00000100:00000001:0.0:1775775499.579456:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775499.579463:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060e00 x1862022710343936/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775499.579471:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775499.579475:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775499.579478:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004a3bddaa] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060e00 x1862022710343936/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775499.579498:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775499.579501:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775499.579504:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775499.579505:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775499.579506:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775499.579508:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775499.579509:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12382 00000400:00000200:0.0:1775775499.579512:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775499.579514:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775499.579516:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775499.579518:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775499.579519:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775499.579520:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775499.579522:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775499.579524:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775499.579525:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775499.579527:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775499.579530:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775499.579532:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775499.579534:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775499.579536:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775499.579538:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775499.579542:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775499.579545:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775499.579551:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775499.579559:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775499.579561:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775499.579566:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775499.579571:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775499.579573:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775499.579586:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775499.579595:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775499.579601:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775499.579611:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775499.579622:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063100. 00000020:00000040:0.0:1775775499.579627:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775499.579632:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775499.579637:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775499.579648:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775499.579653:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775499.579655:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775499.579659:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4800. 00000100:00000001:0.0:1775775499.579663:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775499.579671:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063100 x1862022710344064/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775499.579687:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775499.579692:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775499.579698:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033f6db7f] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063100 x1862022710344064/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775499.579729:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775499.579735:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775499.579754:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775499.579757:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775499.579765:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775499.579770:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775499.579777:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12382 00000400:00000200:0.0:1775775499.579786:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775499.579796:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775499.579799:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775499.579802:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775499.579810:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775499.579818:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775499.579821:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775499.579828:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775499.579831:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775499.579836:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775499.579839:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775499.579841:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775499.579850:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775499.579862:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775499.579869:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775499.579886:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775499.579893:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775499.579902:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775499.579910:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775499.579915:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775499.579919:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775499.579923:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775499.579931:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775499.579935:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775499.579941:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775499.579945:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775499.579948:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775499.579956:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061c00. 00000020:00000040:0.0:1775775499.579959:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775499.579961:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775499.579962:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775499.579964:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775499.579969:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775499.579971:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775499.579976:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5c00. 00000100:00000001:0.0:1775775499.579983:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775499.579996:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061c00 x1862022710344192/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775499.580011:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775499.580017:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775499.580018:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:0.0:1775775499.580024:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076a44879] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061c00 x1862022710344192/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1775775499.580028:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000100:00000001:0.0:1775775499.580042:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775499.580050:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775499.580060:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775499.580063:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775499.580072:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775499.580085:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12392) 00000800:00000200:1.1:1775775499.580431:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775499.580433:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775775499.580652:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775499.580654:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00010000:00000001:3.0:1775775499.707222:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775499.707227:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775499.707231:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775499.707232:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775499.707234:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775499.707235:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775500.731254:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775500.731259:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775500.731264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775500.731266:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775500.731270:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775500.731271:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775501.755240:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775501.755245:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775501.755249:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775501.755250:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775501.755253:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775501.755254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775501.755306:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775501.755309:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775501.755312:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775502.779288:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775502.779299:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775502.779302:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775502.779307:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775502.779313:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775502.779316:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775502.779319:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775502.779322:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775502.779327:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775502.779329:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775502.779333:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775502.779335:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775503.483285:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307058439, at 4307058496 00000100:00000010:0.0:1775775503.483296:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ba0. 00000100:00000001:2.0:1775775503.675285:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775503.675290:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775503.675292:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775503.675296:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775503.675300:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775503.675304:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775503.675306:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775503.675309:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775503.675312:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775503.803260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775503.803266:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775503.803271:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775503.803272:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775503.803276:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775503.803277:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775503.803302:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775503.803304:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775503.803307:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775504.699242:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775775504.699244:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1775775504.699252:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775504.699255:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775504.699260:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775504.699260:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775504.699264:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775504.699265:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00100000:2.0:1775775504.699270:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775504.699274:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775775504.699275:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12387 00000100:00000001:2.0:1775775504.699279:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775504.699282:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775504.699284:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775504.699287:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775504.699293:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775504.699300:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775504.699303:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775504.699307:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775504.699311:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775504.699313:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775504.699315:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775504.699319:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775504.699321:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775504.699323:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775504.699326:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775504.699328:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775504.699329:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775504.699331:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775504.699332:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775504.699334:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775504.699335:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775504.699337:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775504.699340:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775504.699342:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775504.699345:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775504.699349:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775504.699352:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775504.699353:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775504.699356:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775504.699365:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061180. 00000020:00000040:0.0:1775775504.699368:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775504.699372:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775504.699374:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775504.699376:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775504.699379:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775504.699384:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775504.699394:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5c00. 00000100:00000001:0.0:1775775504.699400:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775504.699413:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061180 x1862022710344320/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775504.699426:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775504.699432:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775504.699439:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e23859] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061180 x1862022710344320/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775504.699466:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775504.699469:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775504.699473:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775504.699474:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775504.699476:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775504.699478:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775504.699480:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12387 00000400:00000200:0.0:1775775504.699483:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775504.699486:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775504.699488:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775504.699490:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775504.699492:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775504.699495:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775504.699496:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775504.699499:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775504.699500:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775504.699501:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775504.699503:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775504.699505:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775504.699506:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775504.699508:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775504.699509:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775504.699511:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775504.699512:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775504.699518:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775504.699520:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775504.699524:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775504.699527:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775504.699533:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775504.699535:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775504.699538:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775504.699545:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775504.699550:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775504.699553:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775504.699557:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061880. 00000020:00000040:0.0:1775775504.699562:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775504.699565:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775504.699567:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775504.699569:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775504.699571:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775504.699574:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775504.699578:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4800. 00000100:00000001:0.0:1775775504.699584:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775504.699599:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061880 x1862022710344448/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775504.699611:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775504.699613:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775504.699616:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005afd1f27] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061880 x1862022710344448/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775504.699629:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775504.699632:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775504.699637:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775504.699640:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775504.699641:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775504.699644:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775504.699646:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12387 00000400:00000200:0.0:1775775504.699649:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775504.699651:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775504.699656:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775504.699659:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775504.699660:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775504.699662:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775504.699664:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775504.699669:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775504.699670:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775504.699671:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775504.699672:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775504.699673:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775504.699675:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775504.699683:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775504.699686:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775504.699690:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775504.699692:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775504.699696:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775504.699701:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775504.699706:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775504.699710:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775504.699714:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775504.699718:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775504.699720:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775504.699722:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775504.699725:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775504.699726:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775504.699729:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060700. 00000020:00000040:0.0:1775775504.699730:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775504.699733:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775504.699736:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775504.699740:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775504.699741:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775504.699742:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775504.699746:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7c00. 00000100:00000001:0.0:1775775504.699750:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775504.699757:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060700 x1862022710344576/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775504.699767:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775504.699770:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775504.699773:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000225c8695] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060700 x1862022710344576/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775504.699796:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775504.699799:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775504.699804:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775504.699810:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775504.699813:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775504.699819:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12397) 00000800:00000200:1.1:1775775504.699900:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775504.699904:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775775504.700374:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775504.700376:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775775504.700604:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775504.700606:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00010000:00000001:3.0:1775775504.827260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775504.827267:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775504.827273:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775504.827274:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775504.827278:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775504.827279:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775505.723227:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775505.723234:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775505.723236:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775505.723241:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775505.723245:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775505.723248:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775505.723252:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775505.723254:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775505.723257:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775505.851229:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775505.851234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775505.851238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775505.851239:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775505.851242:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775505.851243:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775506.747232:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775506.747238:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775506.747240:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775506.747244:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775506.747247:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775506.747250:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775506.747252:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775506.747254:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775506.747256:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775506.875243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775506.875248:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775506.875252:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775506.875253:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775506.875256:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775506.875257:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775506.875315:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775506.875318:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775506.875321:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775507.899210:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775507.899214:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775507.899219:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775507.899221:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775507.899223:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775507.899226:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775507.899228:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775507.899230:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775507.899232:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775507.899233:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775507.899236:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775507.899237:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775507.899238:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775507.899239:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775507.899240:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775507.899242:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775507.899243:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775507.899244:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775508.603236:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307063558, at 4307063616 00000100:00000010:0.0:1775775508.603244:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e27e0. 00010000:00000001:3.0:1775775508.923211:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775508.923217:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775508.923223:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775508.923225:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775508.923229:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775508.923231:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775508.923254:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775508.923257:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775508.923260:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775509.819248:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775509.819256:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775509.819259:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775509.819263:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775509.819268:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775509.819275:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12392 00000400:00000200:3.0:1775775509.819284:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775509.819289:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775509.819291:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775509.819294:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775509.819296:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775509.819298:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775509.819300:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775509.819303:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775509.819304:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775509.819306:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775509.819309:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775509.819310:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775509.819312:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775509.819315:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775509.819316:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775509.819317:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775509.819319:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775509.819321:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775509.819323:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775509.819324:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775509.819327:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775509.819329:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775509.819331:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775509.819337:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775509.819339:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775509.819345:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775775509.819348:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775509.819351:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775509.819352:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775509.819355:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775509.819357:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775509.819360:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775509.819366:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000100:00000001:3.0:1775775509.819371:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775509.819378:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710344704/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775509.819386:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775509.819389:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775509.819393:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710344704/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775509.819408:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775509.819410:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775509.819412:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775509.819413:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775509.819415:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775509.819417:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775509.819419:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12392 00000400:00000200:3.0:1775775509.819422:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775509.819425:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775509.819427:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775509.819430:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775509.819431:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775509.819434:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775509.819437:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775509.819440:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775509.819443:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775509.819444:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775509.819447:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775509.819449:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775509.819453:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775509.819459:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775509.819464:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775509.819468:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775509.819473:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775509.819475:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775509.819479:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775509.819483:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775509.819484:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775509.819492:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775509.819495:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775509.819500:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775509.819503:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775509.819506:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775509.819509:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775509.819513:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000020:00000040:3.0:1775775509.819516:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775509.819520:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775509.819523:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775509.819524:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775509.819525:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775509.819526:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775509.819529:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000100:00000001:3.0:1775775509.819532:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775509.819537:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710344832/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775509.819543:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775509.819546:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775509.819551:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710344832/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775509.819568:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775509.819571:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775509.819576:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775509.819579:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775509.819582:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775509.819585:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775509.819589:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12392 00000400:00000200:3.0:1775775509.819598:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775509.819603:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775509.819608:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775509.819612:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775509.819616:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775509.819620:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775509.819624:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775509.819626:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775509.819629:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775509.819630:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775509.819633:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775509.819637:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775509.819639:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775509.819642:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775509.819646:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775509.819650:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775509.819652:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775509.819655:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775509.819661:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775509.819663:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775509.819666:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775509.819669:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775509.819674:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775509.819678:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775509.819680:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775509.819685:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775509.819689:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775509.819694:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775775509.819708:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775509.819714:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775509.819716:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775509.819719:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775509.819723:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775509.819727:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775509.819733:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000100:00000001:3.0:1775775509.819738:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775509.819744:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710344960/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775509.819758:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775509.819761:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775509.819768:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710344960/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1775775509.819769:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775509.819775:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000100:00000001:3.0:1775775509.819787:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775509.819792:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775509.819795:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775509.819800:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775509.819807:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775509.819813:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12402) 00000800:00000200:1.1:1775775509.820091:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775509.820093:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775775509.820299:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775509.820300:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00010000:00000001:3.0:1775775509.947247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775509.947252:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775509.947256:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775509.947258:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775509.947260:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775509.947261:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775510.971253:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775510.971259:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775510.971266:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775510.971268:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775510.971272:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775510.971273:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775511.995206:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775511.995212:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775511.995218:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775511.995219:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775511.995223:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775511.995225:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775511.995288:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775511.995292:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775511.995296:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775513.019228:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775513.019234:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775513.019236:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775513.019238:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775513.019242:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775513.019244:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775513.019246:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775513.019248:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775513.019250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775513.019251:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775513.019254:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775513.019255:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775513.723251:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307068678, at 4307068736 00000100:00000010:0.0:1775775513.723261:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2e40. 00010000:00000001:3.0:1775775514.043216:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775514.043221:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775514.043225:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775514.043226:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775514.043229:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775514.043230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775514.043251:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775514.043252:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775514.043255:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775514.939282:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775514.939288:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775514.939291:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775514.939295:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775514.939299:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775514.939306:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12397 00000400:00000200:2.0:1775775514.939316:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775514.939322:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775514.939325:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775514.939329:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775514.939332:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775514.939334:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775514.939337:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775514.939341:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775514.939343:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775514.939346:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775514.939349:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775514.939352:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775514.939354:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775514.939357:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775514.939359:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775514.939362:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775514.939364:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775514.939367:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775514.939369:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775514.939372:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775514.939376:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775514.939379:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775514.939382:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775514.939384:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775514.939386:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775514.939395:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775775514.939399:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775514.939403:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775514.939404:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775514.939407:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775514.939409:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775514.939414:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775514.939421:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775514.939428:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775514.939437:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710345088/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775514.939447:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775514.939452:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775514.939457:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710345088/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775514.939479:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775514.939483:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775514.939487:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775514.939489:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775514.939492:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775514.939496:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775514.939499:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12397 00000400:00000200:2.0:1775775514.939506:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775514.939513:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775514.939519:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775514.939526:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775514.939532:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775514.939538:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775514.939543:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775514.939548:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775514.939552:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775514.939555:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775514.939559:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775514.939563:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775514.939568:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775514.939577:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775514.939580:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775514.939583:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775514.939588:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775514.939593:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775514.939600:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775514.939605:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775514.939624:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775514.939630:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775514.939632:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775514.939635:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775514.939641:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775514.939648:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775514.939653:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775514.939661:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775775514.939666:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775514.939672:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775514.939676:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775514.939679:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775514.939682:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775514.939687:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775514.939696:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775775514.939703:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775514.939713:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710345216/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775514.939730:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775514.939733:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775514.939742:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710345216/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775514.939762:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775514.939768:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775514.939777:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775514.939779:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775514.939781:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775514.939785:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775514.939791:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12397 00000400:00000200:2.0:1775775514.939800:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000200:1.1:1775775514.939805:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000400:00000200:2.0:1775775514.939809:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775514.939813:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775514.939816:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775514.939818:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775514.939820:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775514.939824:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775514.939829:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775514.939832:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775514.939836:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775514.939839:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775514.939844:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775514.939848:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775514.939853:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775514.939857:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775514.939861:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775514.939865:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775514.939867:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775514.939871:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775514.939876:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775514.939880:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775514.939885:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775514.939888:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775514.939894:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775514.939897:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775514.939902:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775514.939905:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775514.939911:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775775514.939916:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775514.939922:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775514.939925:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775514.939928:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775514.939931:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775514.939935:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775514.939943:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775775514.939950:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775514.939959:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710345344/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775514.939990:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775514.939994:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775514.940001:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710345344/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775514.940023:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775514.940028:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775514.940037:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775514.940042:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775514.940045:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775514.940051:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12407) 00000800:00000200:1.1:1775775514.940160:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775514.940333:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00010000:00000001:3.0:1775775515.067234:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775515.067240:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775515.067247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775515.067248:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775515.067252:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775515.067254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775516.091278:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775516.091288:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775516.091295:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775516.091297:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775516.091301:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775516.091303:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775517.115268:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775517.115273:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775517.115279:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775517.115281:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775517.115284:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775517.115285:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775517.115337:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775517.115340:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775517.115343:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775775518.011517:0:2402:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:0.0:1775775518.011581:0:2402:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1775775518.011593:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775518.011597:0:2402:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:0.0:1775775518.011604:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775518.011608:0:2402:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:0.0:1775775518.011615:0:2402:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:0.0:1775775518.011619:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:0.0:1775775518.011624:0:2402:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775775518.011629:0:2402:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775775518.011635:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3dd0 00000400:00000010:0.0:1775775518.011638:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3dd0. 00000100:00000001:0.0:1775775518.011645:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775518.011654:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6233449f80 x1862022710345088/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775539 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775518.011691:0:2402:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775518.011697:0:2402:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449f80 x1862022710345088/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775539 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775518.011715:0:2402:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775518.011717:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775775518.011726:0:2402:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:0.0:1775775518.011727:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230000. 00000400:00000200:0.0:1775775518.011729:0:2402:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775775518.011732:0:2402:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775775518.011734:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3990 00000400:00000010:0.0:1775775518.011735:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3990. 00000100:00000001:0.0:1775775518.011737:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775518.011740:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6233448380 x1862022710345216/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775539 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775518.011747:0:2402:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775518.011750:0:2402:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448380 x1862022710345216/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775539 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775518.011757:0:2402:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775518.011758:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775775518.011763:0:2402:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:0.0:1775775518.011766:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230c00. 00000400:00000200:0.0:1775775518.011768:0:2402:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775775518.011772:0:2402:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775775518.011775:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3330 00000400:00000010:0.0:1775775518.011777:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3330. 00000100:00000001:0.0:1775775518.011779:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775518.011784:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f623344aa00 x1862022710345344/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775539 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775518.011793:0:2402:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775518.011796:0:2402:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344aa00 x1862022710345344/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775539 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775518.011803:0:2402:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775518.011806:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:3.0:1775775518.139284:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775518.139291:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775518.139297:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775518.139301:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775518.139303:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775518.139306:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775518.139310:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775518.139312:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775518.139314:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775518.139315:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775518.139319:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775518.139320:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775518.139321:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775518.139322:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775518.139323:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775518.139324:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775518.139325:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775518.139326:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775518.843266:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307073798, at 4307073856 00000100:00000010:0.0:1775775518.843282:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e28a0. 00010000:00000001:3.0:1775775519.163256:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775519.163263:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775519.163269:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775519.163270:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775519.163273:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775519.163274:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775519.163298:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775519.163300:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775519.163303:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775520.059225:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775520.059231:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775520.059234:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775520.059238:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775520.059243:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775520.059250:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12402 00000400:00000200:0.0:1775775520.059259:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775520.059265:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775520.059269:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775520.059273:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775520.059276:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775520.059279:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775520.059281:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775520.059285:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775520.059287:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775520.059290:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775520.059293:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775520.059295:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775520.059297:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775520.059300:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775520.059303:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775520.059305:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775520.059306:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775520.059309:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775520.059312:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775520.059315:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775520.059318:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775520.059321:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775520.059324:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775520.059327:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775520.059329:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775520.059336:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061f80. 00000020:00000040:0.0:1775775520.059339:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775520.059344:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775520.059345:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775520.059348:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775520.059351:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775520.059356:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775520.059363:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7000. 00000100:00000001:0.0:1775775520.059370:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775520.059379:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061f80 x1862022710345472/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775520.059389:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775520.059394:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775520.059399:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000029d3794] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061f80 x1862022710345472/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775520.059429:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775520.059433:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775520.059437:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775520.059439:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775520.059441:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775520.059444:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775520.059446:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12402 00000400:00000200:0.0:1775775520.059450:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775520.059454:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775520.059456:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775520.059459:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775520.059461:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775520.059463:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775520.059466:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775520.059468:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775520.059470:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775520.059471:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775520.059474:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775520.059476:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775520.059478:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775520.059481:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775520.059483:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775520.059484:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775520.059487:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775520.059489:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775520.059491:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775520.059494:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775520.059496:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775520.059501:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775520.059503:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775520.059505:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775520.059509:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775520.059512:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775520.059514:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775520.059547:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063b80. 00000020:00000040:0.0:1775775520.059551:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775520.059554:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775520.059555:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775520.059557:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775520.059559:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775520.059561:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775520.059565:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5800. 00000100:00000001:0.0:1775775520.059569:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775520.059575:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063b80 x1862022710345600/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775520.059590:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775520.059592:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775520.059596:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ee9007b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063b80 x1862022710345600/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775520.059605:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775520.059608:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775520.059611:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775520.059613:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775520.059614:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775520.059618:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775520.059621:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12402 00000400:00000200:0.0:1775775520.059625:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775520.059628:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775520.059630:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775520.059633:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775520.059636:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775520.059637:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775520.059640:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775520.059642:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775520.059644:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775520.059645:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775520.059647:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775520.059650:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775520.059652:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775520.059655:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775520.059657:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775520.059659:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775520.059661:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775520.059664:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775520.059667:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775520.059670:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775520.059672:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775520.059675:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775520.059676:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775520.059679:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775520.059681:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775520.059683:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775520.059685:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775520.059689:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061500. 00000020:00000040:0.0:1775775520.059691:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775520.059694:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775520.059695:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775520.059697:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775520.059698:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775520.059700:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775520.059703:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6400. 00000100:00000001:0.0:1775775520.059707:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775520.059711:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061500 x1862022710345728/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775520.059719:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775520.059721:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775520.059725:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb8d992] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061500 x1862022710345728/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775520.059732:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775520.059735:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775520.059739:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775520.059742:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775520.059746:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775520.059753:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12412) 00000800:00000200:1.1:1775775520.060026:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775520.060030:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775775520.060409:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775520.060411:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775775520.060746:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775520.060748:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00010000:00000001:3.0:1775775520.187249:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775520.187256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775520.187263:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775520.187264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775520.187268:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775520.187269:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775521.211237:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775521.211246:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775521.211255:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775521.211256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775521.211261:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775521.211263:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775522.235275:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775522.235281:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775522.235286:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775522.235287:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775522.235291:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775522.235293:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775522.235368:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775522.235372:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775522.235376:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775523.259245:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775523.259251:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775523.259253:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775523.259255:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775523.259259:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775523.259260:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775523.259262:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775523.259263:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775523.259266:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775523.259267:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775523.259270:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775523.259271:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775523.963227:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307078918, at 4307078976 00000100:00000010:0.0:1775775523.963241:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2a20. 00010000:00000001:3.0:1775775524.283223:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775524.283234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775524.283238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775524.283239:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775524.283242:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775524.283243:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775524.283265:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775524.283266:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775524.283268:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775525.179261:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775525.179271:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775525.179275:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775525.179280:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775525.179286:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775525.179295:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12407 00000400:00000200:3.0:1775775525.179305:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775525.179312:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775525.179316:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775525.179319:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775525.179322:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775525.179324:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775525.179325:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775525.179329:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775525.179330:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775525.179333:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775525.179335:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775525.179337:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775525.179339:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775525.179340:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775525.179342:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775525.179343:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775525.179344:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775525.179347:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775525.179349:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775525.179351:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775525.179354:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775525.179356:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775525.179358:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775525.179359:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775525.179361:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775525.179368:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73100. 00000020:00000040:3.0:1775775525.179371:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775525.179375:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775525.179376:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775525.179379:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775525.179381:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775525.179385:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775525.179394:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000100:00000001:3.0:1775775525.179400:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775525.179410:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73100 x1862022710345856/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775525.179446:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775525.179452:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775525.179457:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f67e6552] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73100 x1862022710345856/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775525.179480:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775525.179483:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775525.179485:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775525.179486:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775525.179488:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775525.179490:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775525.179492:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12407 00000400:00000200:3.0:1775775525.179496:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775525.179500:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775525.179502:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775525.179504:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775525.179506:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775525.179509:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775525.179512:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775525.179515:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775525.179516:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775525.179519:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775525.179521:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775525.179525:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775525.179530:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775525.179537:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775525.179539:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775525.179543:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775525.179546:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775525.179549:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775525.179551:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775525.179554:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775525.179556:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775525.179562:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775525.179566:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775525.179568:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775525.179573:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775525.179576:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775525.179578:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775525.179582:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775775525.179585:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775525.179587:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775525.179588:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775525.179590:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775525.179594:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775525.179596:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775525.179603:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000100:00000001:3.0:1775775525.179610:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775525.179619:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710345984/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775525.179634:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775525.179637:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775525.179645:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710345984/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775525.179663:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775525.179666:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775525.179670:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775525.179674:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775525.179676:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775525.179682:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775525.179686:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12407 00000400:00000200:3.0:1775775525.179694:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775525.179699:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775525.179701:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775525.179708:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775525.179711:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775525.179712:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775525.179714:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775525.179718:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775525.179721:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775525.179723:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775525.179724:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775525.179727:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775525.179731:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775525.179736:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775525.179740:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775525.179741:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775525.179744:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775525.179746:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775525.179751:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775525.179754:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775525.179755:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775525.179759:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775525.179763:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775525.179766:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775525.179768:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775525.179771:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775525.179773:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775525.179777:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000020:00000040:3.0:1775775525.179781:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775525.179786:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775525.179788:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775525.179790:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775525.179791:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775525.179793:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775525.179796:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000100:00000001:3.0:1775775525.179799:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775525.179806:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710346112/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775525.179819:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775525.179823:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775525.179828:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710346112/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775525.179840:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775525.179846:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775525.179853:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775525.179859:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775525.179865:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775775525.179871:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775775525.179874:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12417) 00000800:00000200:1.1:1775775525.179876:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775775525.180329:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775525.180331:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775775525.180634:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775525.180636:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00010000:00000001:3.0:1775775525.307250:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775525.307256:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775525.307263:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775525.307264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775525.307269:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775525.307270:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775526.331226:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775526.331230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775526.331234:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775526.331235:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775526.331238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775526.331238:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775527.355264:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775527.355270:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775527.355275:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775527.355279:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775527.355281:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775527.355285:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775527.355289:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775527.355291:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775527.355293:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775527.355296:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775527.355301:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775527.355303:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775527.355304:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775527.355305:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775527.355307:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775527.355308:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775527.355310:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775527.355312:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775527.355377:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775527.355382:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775527.355386:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775528.379292:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775528.379299:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775528.379304:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775528.379306:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775528.379309:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775528.379310:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775775529.083220:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:0.0:1775775529.083225:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:0.0:1775775529.083230:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000100:00000040:0.0:1775775529.083253:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307084038, at 4307084096 00000100:00000010:0.0:1775775529.083258:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e28a0. 00000800:00000001:0.0:1775775529.083334:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775775529.083347:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775775529.083353:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775775529.083355:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1775775529.403282:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775529.403288:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775529.403294:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775529.403295:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775529.403298:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775529.403299:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775529.403325:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775529.403327:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775529.403330:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775530.299256:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775530.299266:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775530.299270:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775530.299275:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775530.299282:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775530.299292:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12412 00000400:00000200:3.0:1775775530.299302:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775530.299311:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775530.299314:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775530.299318:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775530.299321:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775530.299323:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775530.299325:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775530.299328:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775530.299329:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775530.299332:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775530.299334:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775530.299336:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775530.299338:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775530.299352:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775530.299354:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775530.299355:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775530.299361:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775530.299363:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775530.299366:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775530.299368:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775530.299371:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775530.299374:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775530.299376:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775530.299378:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775530.299380:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775530.299390:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775775530.299394:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775530.299397:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775530.299399:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775530.299401:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775530.299404:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775530.299409:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775530.299417:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000100:00000001:3.0:1775775530.299424:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775530.299434:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710346240/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775530.299444:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775530.299448:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775530.299452:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710346240/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775530.299471:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775530.299473:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775530.299476:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775530.299478:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775530.299479:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775530.299482:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775530.299483:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12412 00000400:00000200:3.0:1775775530.299486:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775530.299489:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775530.299491:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775530.299493:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775530.299495:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775530.299498:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775530.299501:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775530.299503:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775530.299505:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775530.299506:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775530.299508:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775530.299509:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775530.299511:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775530.299513:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775530.299519:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775530.299522:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775530.299528:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775530.299533:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775530.299537:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775530.299540:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775530.299542:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775530.299548:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775530.299553:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775530.299559:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775530.299566:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775530.299575:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775530.299584:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775530.299588:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775775530.299592:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775530.299595:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775530.299597:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775530.299602:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775530.299604:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775530.299607:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775530.299613:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:3.0:1775775530.299618:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775530.299630:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710346368/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775530.299640:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775530.299645:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775530.299652:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710346368/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775530.299662:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775530.299664:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775530.299670:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775530.299674:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775530.299678:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775530.299680:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775530.299685:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12412 00000400:00000200:3.0:1775775530.299693:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775530.299700:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775530.299702:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775530.299707:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775530.299709:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775530.299714:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775530.299716:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775530.299719:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775530.299723:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775530.299729:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775530.299734:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775530.299738:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775530.299744:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775530.299761:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775530.299769:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775530.299777:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775530.299783:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775530.299794:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775530.299801:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775530.299812:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775530.299814:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775530.299820:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775530.299827:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775530.299833:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775530.299840:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775530.299851:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775530.299861:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775530.299869:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775775530.299877:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775530.299881:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775530.299889:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775530.299894:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775530.299897:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775530.299900:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775530.299908:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000800:00000200:1.1:1775775530.299916:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775775530.299918:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775530.299920:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000100:00100000:3.0:1775775530.299933:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710346496/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775530.299971:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775530.299979:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775530.299986:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710346496/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775530.300060:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775530.300066:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775530.300078:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775530.300090:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775530.300103:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775530.300112:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12423) 00000800:00000200:1.1:1775775530.300395:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775530.300397:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1775775530.300621:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775530.300622:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00010000:00000001:3.0:1775775530.427286:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775530.427292:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775530.427297:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775530.427299:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775530.427301:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775530.427302:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775531.451216:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775531.451222:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775531.451229:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775531.451230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775531.451234:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775531.451235:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775532.475219:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775532.475227:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775532.475231:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775532.475235:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775532.475240:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775532.475242:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775532.475245:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775532.475247:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775532.475252:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775532.475253:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775532.475257:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775532.475258:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775532.475344:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775532.475349:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775532.475352:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775533.499216:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775533.499221:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775533.499225:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775533.499226:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775533.499229:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775533.499230:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775534.203238:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307089158, at 4307089216 00000100:00000010:0.0:1775775534.203248:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2e40. 00010000:00000001:3.0:1775775534.523243:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775534.523248:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775534.523253:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775534.523254:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775534.523256:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775534.523257:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775534.523291:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775534.523293:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775534.523295:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775535.419240:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775535.419246:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775535.419259:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775535.419263:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775535.419266:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775535.419274:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12418 00000400:00000200:0.0:1775775535.419284:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775535.419289:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775535.419293:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775535.419296:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775535.419299:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775535.419301:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775535.419303:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775535.419306:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775535.419307:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775535.419310:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775535.419313:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775535.419314:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775535.419316:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775535.419318:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775535.419320:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775535.419321:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775535.419323:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775535.419325:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775535.419327:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775535.419329:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775535.419332:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775535.419334:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775535.419337:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775535.419338:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775535.419340:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775535.419349:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062a00. 00000020:00000040:0.0:1775775535.419352:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775535.419356:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775535.419357:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775535.419360:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775535.419362:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775535.419366:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775535.419374:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6000. 00000100:00000001:0.0:1775775535.419380:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775535.419390:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062a00 x1862022710346624/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775535.419400:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775535.419405:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775535.419408:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009753f9f6] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062a00 x1862022710346624/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775535.419427:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775535.419429:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775535.419432:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775535.419433:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775535.419434:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775535.419436:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775535.419439:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12418 00000400:00000200:0.0:1775775535.419441:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775535.419444:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775535.419446:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775535.419449:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775535.419452:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775535.419456:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775535.419460:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775535.419465:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775535.419467:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775535.419472:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775535.419476:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775535.419481:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775535.419485:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775535.419495:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775535.419500:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775535.419506:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775535.419511:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775535.419517:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775535.419521:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775535.419523:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775535.419527:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775535.419535:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775535.419538:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775535.419540:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775535.419545:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775535.419548:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775535.419550:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775535.419555:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062d80. 00000020:00000040:0.0:1775775535.419557:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775535.419558:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775535.419559:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775535.419562:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775535.419565:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775535.419567:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775535.419572:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7800. 00000100:00000001:0.0:1775775535.419578:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775535.419593:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062d80 x1862022710346752/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775535.419609:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775535.419612:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775535.419619:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009965f80b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062d80 x1862022710346752/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775535.419634:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775535.419639:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775535.419647:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775535.419650:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775535.419652:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775535.419658:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775535.419663:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12418 00000400:00000200:0.0:1775775535.419710:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775535.419715:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775535.419718:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775535.419720:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775535.419722:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775535.419724:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775535.419726:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775535.419728:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775535.419731:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775535.419733:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775535.419735:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775535.419736:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775535.419739:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775535.419743:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775535.419747:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775535.419750:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775535.419753:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775535.419755:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000800:00000200:1.1:1775775535.419759:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00080000:0.0:1775775535.419760:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775535.419764:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775535.419765:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 02000000:00000001:0.0:1775775535.419767:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775535.419770:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775535.419774:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775535.419777:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775535.419779:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775535.419782:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775535.419784:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775535.419790:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062680. 00000020:00000040:0.0:1775775535.419796:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775535.419801:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775535.419804:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775535.419807:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775535.419809:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775535.419812:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775535.419819:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000001:0.0:1775775535.419825:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775535.419835:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062680 x1862022710346880/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775535.419849:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775535.419852:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775535.419859:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000082c7985e] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062680 x1862022710346880/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775535.419869:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775535.419873:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775535.419875:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775535.419878:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775535.419882:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775535.419886:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12428) 00000800:00000200:1.1:1775775535.420168:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775535.420169:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775775535.420387:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775535.420389:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00010000:00000001:3.0:1775775535.547292:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775535.547300:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775535.547305:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775535.547306:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775535.547309:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775535.547310:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775536.571223:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775536.571228:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775536.571246:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775536.571247:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775536.571249:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775536.571250:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775537.595259:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775537.595264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775537.595268:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775537.595271:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775537.595273:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775537.595276:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775537.595278:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775537.595280:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775537.595281:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775537.595283:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775537.595286:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775537.595287:0:214474:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775537.595288:0:214474:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775537.595288:0:214474:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775537.595289:0:214474:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775537.595290:0:214474:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775537.595291:0:214474:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775537.595293:0:214474:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775537.595350:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775537.595354:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775537.595358:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775538.619247:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775538.619252:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775538.619258:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775538.619260:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775538.619263:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775538.619264:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775539.323307:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307094278, at 4307094336 00000100:00000010:0.0:1775775539.323318:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e27e0. 00010000:00000001:3.0:1775775539.643301:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775539.643311:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775539.643319:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775539.643320:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775539.643325:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775539.643327:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775539.643361:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775539.643365:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775539.643369:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775540.539301:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775775540.539304:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775540.539310:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775540.539312:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775540.539313:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775540.539318:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1775775540.539318:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775540.539323:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775775540.539325:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775540.539326:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775540.539331:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775540.539333:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775775540.539333:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775540.539338:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775540.539344:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12423 00000400:00000200:0.0:1775775540.539356:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775540.539363:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775540.539368:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775540.539372:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775540.539375:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775540.539378:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775540.539380:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775540.539385:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775540.539387:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775540.539390:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775540.539393:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775540.539395:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775540.539397:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775540.539401:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775540.539403:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775540.539405:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775540.539407:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775540.539410:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775540.539414:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775540.539416:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775540.539420:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775540.539424:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775540.539427:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775540.539429:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775540.539432:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775540.539442:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063800. 00000020:00000040:0.0:1775775540.539446:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775540.539450:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775540.539451:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775540.539455:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775540.539458:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775540.539462:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775540.539471:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4400. 00000100:00000001:0.0:1775775540.539479:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775540.539489:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063800 x1862022710347008/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775540.539500:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775540.539504:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775540.539509:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000015d81d25] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063800 x1862022710347008/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775540.539531:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775540.539535:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775540.539538:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775540.539540:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775540.539542:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775540.539545:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775540.539548:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12423 00000400:00000200:0.0:1775775540.539552:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775540.539556:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775540.539558:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775540.539561:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775540.539563:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775540.539565:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775540.539568:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775540.539573:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775540.539575:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775540.539579:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775540.539583:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775540.539588:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775540.539595:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775540.539604:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775540.539608:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775540.539615:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775540.539620:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775540.539626:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775540.539632:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775540.539638:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775540.539640:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775540.539647:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775540.539652:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775540.539655:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775540.539661:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775540.539667:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775540.539671:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775540.539679:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060380. 00000020:00000040:0.0:1775775540.539682:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775540.539687:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775540.539689:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775540.539693:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775540.539696:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775540.539699:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775540.539706:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000001:0.0:1775775540.539718:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775540.539727:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060380 x1862022710347136/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775540.539738:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775540.539741:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775540.539746:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000614ace82] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060380 x1862022710347136/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775540.539759:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775540.539761:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775540.539777:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775540.539779:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775540.539781:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775540.539788:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775540.539794:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12423 00000400:00000200:0.0:1775775540.539803:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775540.539811:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775540.539816:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000800:00000200:1.1:1775775540.539829:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775540.539833:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000100:00000040:0.0:1775775540.539836:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775540.539842:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775540.539848:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775540.539852:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775540.539855:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775540.539856:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775540.539857:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775540.539859:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775540.539862:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775540.539865:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775540.539869:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775540.539872:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775540.539878:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775540.539881:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775540.539885:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775540.539890:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775540.539893:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775540.539897:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775540.539902:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775540.539908:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775540.539916:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775540.539923:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775540.539933:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775540.539940:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775540.539947:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062680. 00000020:00000040:0.0:1775775540.539951:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775540.539957:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775540.539959:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775540.539962:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775540.539964:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775540.539983:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775540.539991:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7800. 00000100:00000001:0.0:1775775540.539998:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775540.540007:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062680 x1862022710347264/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775540.540023:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775540.540027:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775540.540037:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000082c7985e] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062680 x1862022710347264/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775540.540052:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775540.540057:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775540.540064:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775540.540069:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775540.540078:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775540.540085:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12433) 00000800:00000200:1.1:1775775540.540389:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775540.540392:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775775540.540697:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775540.540700:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00010000:00000001:3.0:1775775540.667229:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775540.667234:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775540.667238:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775540.667239:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775540.667242:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775540.667242:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775541.563257:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775541.563262:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775541.563264:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775541.563268:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775541.563271:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775541.563273:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775541.563276:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775541.563277:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775541.563279:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775541.691271:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775541.691277:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775541.691282:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775541.691284:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775541.691288:0:214474:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775541.691290:0:214474:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775542.587202:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775542.587207:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775542.587208:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775542.587213:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775542.587216:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775542.587218:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775542.587221:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775542.587223:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775542.587225:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775542.715241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775542.715247:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775542.715248:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775542.715251:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775542.715255:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775542.715256:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775542.715258:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775542.715259:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775542.715262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775542.715263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775542.715266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775542.715267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775542.715320:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775542.715323:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775542.715326:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775543.611220:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775543.611226:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775543.611228:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775543.611233:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775543.611235:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775543.611238:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775543.611241:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775543.611242:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775543.611244:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775543.739265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775543.739271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775543.739277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775543.739279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775543.739283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775543.739284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775544.443269:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307099398, at 4307099456 00000100:00000010:0.0:1775775544.443279:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ba0. 00000100:00000001:2.0:1775775544.635222:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775544.635230:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775544.635232:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775544.635238:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775544.635242:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775544.635245:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775544.635248:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775544.635251:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775544.635254:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775544.763214:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775544.763219:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775544.763223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775544.763224:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775544.763227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775544.763227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775544.763242:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775544.763245:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775544.763247:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775545.659212:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775545.659217:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775545.659219:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775545.659224:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775545.659227:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775545.659230:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775545.659232:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775545.659235:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775545.659237:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775545.659247:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775545.659252:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775545.659255:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775545.659258:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775545.659261:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775545.659267:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12428 00000400:00000200:3.0:1775775545.659275:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775545.659279:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775545.659282:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775545.659284:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775545.659286:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775545.659288:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775545.659289:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775545.659292:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775545.659293:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775545.659295:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775545.659298:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775545.659299:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775545.659300:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775545.659302:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775545.659304:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775545.659305:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775545.659307:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775545.659309:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775545.659311:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775545.659313:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775545.659315:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775545.659317:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775545.659319:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775545.659321:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775545.659322:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775545.659330:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775775545.659333:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775545.659336:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775545.659337:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775545.659340:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775545.659342:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775545.659345:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775545.659352:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000001:3.0:1775775545.659357:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775545.659364:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710347392/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775545.659371:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775545.659375:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775545.659378:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710347392/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775545.659395:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775545.659398:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775545.659400:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775545.659401:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775545.659402:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775545.659404:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775545.659406:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12428 00000400:00000200:3.0:1775775545.659408:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775545.659410:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775545.659412:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775545.659413:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775545.659415:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775545.659416:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775545.659417:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775545.659419:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775545.659419:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775545.659420:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775545.659422:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775545.659423:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775545.659424:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775545.659427:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775545.659428:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775545.659430:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775545.659432:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775545.659433:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775545.659435:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775545.659437:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775545.659438:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775545.659443:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775545.659444:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775545.659447:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775545.659451:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775545.659454:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775545.659455:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775545.659458:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775775545.659462:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775545.659468:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775545.659470:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775545.659480:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775545.659482:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775545.659485:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775545.659489:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775775545.659493:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775545.659500:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710347520/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775545.659510:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775545.659513:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775545.659519:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710347520/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775545.659532:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775545.659536:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775545.659539:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775545.659542:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775545.659543:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775545.659545:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775545.659547:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12428 00000400:00000200:3.0:1775775545.659553:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775545.659557:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775545.659560:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775545.659565:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775545.659569:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775545.659574:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775545.659579:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775545.659582:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775545.659586:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775545.659589:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775545.659591:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775545.659594:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775545.659600:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775545.659604:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775545.659607:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775545.659611:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775545.659612:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775545.659613:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775545.659618:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775545.659621:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775545.659622:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775545.659626:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775545.659629:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775545.659631:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775545.659634:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775545.659639:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775545.659642:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775545.659647:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000020:00000040:3.0:1775775545.659652:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775545.659655:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775545.659658:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775545.659660:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775545.659662:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775545.659663:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775545.659666:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:3.0:1775775545.659670:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775545.659674:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710347648/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775545.659689:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775545.659693:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775545.659698:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710347648/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775545.659708:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775545.659712:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775545.659718:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775775545.659722:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775775545.659723:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775545.659728:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775775545.659728:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000100:00000040:3.0:1775775545.659734:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12438) 00000800:00000200:1.1:1775775545.660048:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775545.660050:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775775545.660291:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775545.660293:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00010000:00000001:3.0:1775775545.787258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775545.787264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775545.787270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775545.787271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775545.787274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775545.787275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775546.683231:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775546.683237:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775546.683238:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775546.683243:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775546.683246:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775546.683249:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775546.683252:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775546.683254:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775546.683256:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775546.811247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775546.811253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775546.811259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775546.811261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775546.811266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775546.811267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775547.707200:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775547.707206:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775547.707207:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775547.707211:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775547.707214:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775547.707217:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775547.707219:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775547.707221:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775547.707223:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775547.835238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775547.835245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775547.835251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775547.835254:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775547.835256:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775547.835260:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775547.835263:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775547.835266:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775547.835268:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775547.835271:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775547.835275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775547.835277:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775547.835278:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775547.835280:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775547.835282:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775547.835283:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775547.835285:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775547.835287:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775547.835360:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775547.835366:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775547.835370:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775548.859241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775548.859247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775548.859253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775548.859254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775548.859259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775548.859261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775549.563317:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307104518, at 4307104576 00000100:00000010:0.0:1775775549.563331:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e20c0. 00010000:00000001:3.0:1775775549.883228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775549.883234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775549.883238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775549.883239:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775549.883242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775549.883242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775549.883260:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775549.883262:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775549.883264:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775775550.779486:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775550.779496:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775550.779503:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775550.779524:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491aa00 x1862022710347776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.779565:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775550.779570:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775550.779582:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775550.779589:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775550.779600:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775550.779614:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710347776:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775550.779634:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775550.779642:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775550.779648:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775550.779657:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775550.779669:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775550.779677:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775550.779701:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775550.779710:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775550.779723:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775550.779738:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710347776, portal 25 00000100:00000001:0.0:1775775550.779747:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775550.779751:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684369408 : -123823025182208 : ffff8f623491aa00) 00000100:00000040:0.0:1775775550.779772:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491aa00 x1862022710347776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775575 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.779818:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775550.779829:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775550.779833:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775550.779838:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710347776, offset 0 00000400:00000200:0.0:1775775550.779848:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775550.779860:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775550.779869:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775550.779876:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775550.779881:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775550.779887:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775550.779890:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775550.779895:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453269:453269:256:4294967295] 192.168.201.115@tcp LPNI seq info [453194:453194:8:4294967295] 00000400:00000200:0.0:1775775550.779901:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775550.779914:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775550.779922:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775550.779929:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775550.779937:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775550.779942:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775775550.779949:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775550.779953:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1775775550.779958:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775550.779961:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775550.779969:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775550.779975:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775550.779996:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775550.780000:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775550.780006:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775550.780011:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491aa00 x1862022710347776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775575 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780023:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775550.780026:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491aa00 x1862022710347776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775575 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780035:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775550.780037:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775550.780044:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775550.780049:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775550.780052:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307109638 00000100:00000010:0.0:1775775550.780059:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775550.780067:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775550.780076:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307109638 00000100:00000010:0.0:1775775550.780080:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3780. 00000100:00000001:0.0:1775775550.780084:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775550.780088:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775550.780091:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775550.780094:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775550.780098:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775550/real 1775775550] req@ffff8f623491aa00 x1862022710347776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775550.780109:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775550.780110:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775550.780113:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775550.780116:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775550.780117:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775550.780120:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775550.780124:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491aa00 x1862022710347776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775550.780133:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491aa00 x1862022710347776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780176:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775550.780178:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775550.780181:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775550.780184:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775550.780188:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491aa00 x1862022710347776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780196:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775550.780201:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491aa00 x1862022710347776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780210:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775550.780214:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775550.780216:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775550.780217:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775550.780220:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775550.780222:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775550.780225:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775550.780230:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775550.780233:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775550.780237:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491aa00 x1862022710347776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775550.780246:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710347776:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775550.780251:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775550.780253:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775550.780256:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491aa00 x1862022710347776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780264:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775550.780267:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775550.780271:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775550.780274:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775550.780277:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775550.780279:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775550.780282:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775550.780285:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:0.0:1775775550.780290:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775550.780292:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775550.780294:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491aa00. 00000100:00000001:0.0:1775775550.780297:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775550.780299:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775550.780302:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775550.780310:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775550.780312:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775550.780314:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775550.780317:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775550.780318:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775550.780322:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918a80 x1862022710347904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780330:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775550.780331:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775550.780333:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775550.780335:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775550.780338:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775550.780341:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710347904:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775550.780344:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775550.780346:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775550.780348:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775550.780350:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775550.780353:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775550.780356:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775550.780360:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775550.780362:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775550.780366:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775550.780369:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710347904, portal 10 00000100:00000001:0.0:1775775550.780371:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775550.780373:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684361344 : -123823025190272 : ffff8f6234918a80) 00000100:00000040:0.0:1775775550.780380:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918a80 x1862022710347904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775575 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780387:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775550.780389:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775550.780392:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775550.780394:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710347904, offset 0 00000400:00000200:0.0:1775775550.780398:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775550.780402:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775550.780406:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775550.780409:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775550.780412:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775550.780415:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775550.780418:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775550.780421:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453270:453270:256:4294967295] 192.168.201.115@tcp LPNI seq info [453195:453195:8:4294967295] 00000400:00000200:0.0:1775775550.780426:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775550.780433:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775550.780438:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775550.780441:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775550.780445:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775550.780448:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775775550.780452:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775550.780455:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1775775550.780459:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775550.780461:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775550.780464:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775550.780467:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775550.780470:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775550.780472:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775550.780474:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775550.780478:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918a80 x1862022710347904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775575 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780485:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775550.780488:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918a80 x1862022710347904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775575 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780495:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775550.780497:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775550.780500:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775550.780504:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775550.780507:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775550.780509:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307105804 00000100:00000001:0.0:1775775550.780512:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307109638 00000100:00000001:0.0:1775775550.780515:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307105804 00000100:00000001:0.0:1775775550.780518:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307109638 00000100:00000001:0.0:1775775550.780520:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775550.780523:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775550.780525:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775550.780528:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775550/real 1775775550] req@ffff8f6234918a80 x1862022710347904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775550.780536:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775550.780538:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775550.780539:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775550.780541:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775550.780543:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775550.780545:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775550.780548:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918a80 x1862022710347904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775550.780557:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918a80 x1862022710347904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780564:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775550.780565:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775550.780567:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775550.780568:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775550.780571:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918a80 x1862022710347904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780577:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775550.780581:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918a80 x1862022710347904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780587:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775550.780590:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775550.780591:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775550.780593:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775550.780595:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775550.780601:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775550.780604:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775550.780607:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775550.780610:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775550.780613:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918a80 x1862022710347904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775550.780621:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710347904:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775550.780625:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775550.780629:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491b800 x1862022710348032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780635:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775550.780637:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775550.780639:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775550.780641:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775550.780642:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775550.780645:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710348032:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775550.780649:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775550.780651:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775550.780653:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775550.780654:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775550.780657:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775775550.780660:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775550.780663:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775550.780665:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775550.780668:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775550.780671:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710348032, portal 10 00000100:00000001:0.0:1775775550.780673:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775550.780675:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684372992 : -123823025178624 : ffff8f623491b800) 00000100:00000040:0.0:1775775550.780680:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491b800 x1862022710348032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775575 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780687:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775550.780689:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775550.780692:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775550.780694:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710348032, offset 0 00000400:00000200:0.0:1775775550.780698:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775550.780701:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775550.780704:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775550.780707:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775550.780710:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775550.780713:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775550.780715:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775550.780719:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453271:453271:256:4294967295] 192.168.201.115@tcp LPNI seq info [453196:453196:8:4294967295] 00000400:00000200:0.0:1775775550.780723:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775550.780729:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775550.780733:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775550.780737:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775550.780741:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775550.780743:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775775550.780747:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775550.780750:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1775775550.780754:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775550.780756:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775550.780759:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775550.780762:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775550.780765:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775550.780767:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775550.780769:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775550.780772:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491b800 x1862022710348032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775575 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780779:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775550.780782:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491b800 x1862022710348032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775575 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780796:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775550.780797:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775550.780800:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775550.780803:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775550.780806:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775550.780808:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307105804 00000100:00000001:0.0:1775775550.780810:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307109638 00000100:00000001:0.0:1775775550.780813:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307105804 00000100:00000001:0.0:1775775550.780815:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307109638 00000100:00000001:0.0:1775775550.780817:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775550.780819:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775550.780821:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775550.780824:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775550/real 1775775550] req@ffff8f623491b800 x1862022710348032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775550.780830:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775550.780832:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775550.780833:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775550.780835:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775550.780837:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775550.780839:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775550.780842:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491b800 x1862022710348032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775550.780851:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491b800 x1862022710348032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780857:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775550.780858:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775550.780860:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775550.780861:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775550.780865:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491b800 x1862022710348032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780872:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775550.780875:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491b800 x1862022710348032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780882:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775550.780885:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775550.780886:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775550.780888:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775550.780889:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775550.780891:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775550.780894:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775550.780898:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775550.780900:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775550.780905:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491b800 x1862022710348032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775550.780913:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710348032:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775550.780916:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775550.780918:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775550.780921:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918a80 x1862022710347904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780928:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775550.780930:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775550.780931:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775550.780934:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775550.780935:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775550.780937:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775550.780939:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775550.780940:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4c400. 02000000:00000001:0.0:1775775550.780944:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775550.780945:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775550.780947:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918a80. 00000100:00000001:0.0:1775775550.780949:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775550.780951:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775550.780952:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775550.780955:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491b800 x1862022710348032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775575 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775550.780962:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775550.780963:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775550.780964:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775775550.780966:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775550.780967:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775550.780968:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775550.780970:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775550.780971:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:0.0:1775775550.780973:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775550.780975:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775550.780976:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491b800. 00000100:00000001:0.0:1775775550.780979:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775550.780980:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775550.780998:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775550.781001:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775550.781003:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775550.781004:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775550.781006:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775550.907225:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775550.907230:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775550.907235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775550.907236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775550.907239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775550.907241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775550.907286:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775550.907291:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775550.907295:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775551.931252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775551.931256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775551.931260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775551.931261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775551.931264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775551.931265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775552.955219:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775552.955227:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775552.955229:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775552.955232:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775552.955237:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775552.955240:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775552.955242:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775552.955244:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775552.955249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775552.955250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775552.955254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775552.955256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775552.955318:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775552.955322:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775552.955325:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775553.979231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775553.979236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775553.979240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775553.979241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775553.979244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775553.979245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775554.683255:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307109638, at 4307109696 00000100:00000010:0.0:1775775554.683262:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00010000:00000001:3.0:1775775555.003240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775555.003245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775555.003249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775555.003249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775555.003252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775555.003253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775555.003272:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775555.003274:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775555.003277:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775555.899310:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775555.899321:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775555.899324:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775555.899335:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775555.899343:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775555.899348:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775555.899363:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775555.899369:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775555.899379:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775555.899574:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775555.899579:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775555.899584:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775555.899597:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918000 x1862022710348160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.899628:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775555.899629:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775555.899638:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775555.899644:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775555.899651:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775555.899666:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710348160:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775555.899678:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775555.899685:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775555.899691:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775555.899698:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775555.899710:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775775555.899714:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775555.899729:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775555.899736:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775555.899750:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775555.899760:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710348160, portal 25 00000100:00000001:0.0:1775775555.899766:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775555.899770:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684358656 : -123823025192960 : ffff8f6234918000) 00000100:00000040:0.0:1775775555.899784:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918000 x1862022710348160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775580 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.899812:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775555.899817:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775555.899825:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775555.899831:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710348160, offset 0 00000400:00000200:0.0:1775775555.899840:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775555.899853:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775555.899866:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775555.899874:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775555.899881:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775555.899904:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775555.899911:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775555.899919:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453272:453272:256:4294967295] 192.168.201.115@tcp LPNI seq info [453197:453197:8:4294967295] 00000400:00000200:0.0:1775775555.899930:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775555.899952:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775555.899966:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775555.899976:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775555.899996:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775555.900005:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775775555.900024:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775555.900032:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775775555.900051:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775555.900056:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775555.900066:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775555.900074:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775555.900079:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775555.900082:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775555.900088:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775555.900094:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918000 x1862022710348160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775580 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900120:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775555.900128:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918000 x1862022710348160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775580 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900177:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775555.900182:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775555.900188:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775555.900198:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775555.900201:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307114759 00000100:00000010:0.0:1775775555.900210:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775555.900219:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775555.900226:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307114759 00000100:00000010:0.0:1775775555.900230:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3d80. 00000100:00000001:0.0:1775775555.900233:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775555.900236:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775555.900243:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775555.900246:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775555.900251:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775555/real 1775775555] req@ffff8f6234918000 x1862022710348160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775555.900266:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775555.900269:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775555.900275:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775555.900282:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775555.900285:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775555.900291:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775555.900301:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918000 x1862022710348160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775555.900316:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918000 x1862022710348160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900367:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775555.900371:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775555.900381:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775555.900384:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775555.900397:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918000 x1862022710348160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900421:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775555.900429:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918000 x1862022710348160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900450:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775555.900455:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775555.900459:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775555.900461:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775555.900464:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775555.900467:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775555.900472:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775555.900478:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775555.900482:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775555.900488:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918000 x1862022710348160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775555.900497:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710348160:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775555.900502:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775555.900504:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775555.900507:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918000 x1862022710348160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900514:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775555.900517:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775555.900520:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775775555.900524:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775555.900526:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775555.900528:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775555.900531:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775555.900533:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4dc00. 02000000:00000001:0.0:1775775555.900537:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775555.900539:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775555.900541:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918000. 00000100:00000001:0.0:1775775555.900545:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775555.900546:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775555.900549:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775555.900555:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775555.900556:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775555.900558:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775555.900559:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775555.900560:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775555.900564:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491b800 x1862022710348288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900569:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775555.900570:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775555.900572:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775555.900574:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775555.900576:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775555.900578:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710348288:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775555.900581:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775555.900582:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775555.900584:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775555.900585:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775555.900587:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775775555.900588:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775555.900591:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775555.900593:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775555.900596:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775555.900598:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710348288, portal 10 00000100:00000001:0.0:1775775555.900600:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775555.900601:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684372992 : -123823025178624 : ffff8f623491b800) 00000100:00000040:0.0:1775775555.900606:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491b800 x1862022710348288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775580 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900610:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775555.900611:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775555.900613:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775555.900614:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710348288, offset 0 00000400:00000200:0.0:1775775555.900617:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775555.900620:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775555.900622:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775555.900624:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775555.900626:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775555.900628:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775555.900629:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775555.900632:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453273:453273:256:4294967295] 192.168.201.115@tcp LPNI seq info [453198:453198:8:4294967295] 00000400:00000200:0.0:1775775555.900635:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775555.900639:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775555.900642:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775555.900645:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775555.900648:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775555.900650:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775775555.900653:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775555.900655:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775775555.900658:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775555.900659:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775555.900661:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775555.900663:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775555.900665:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775555.900667:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775555.900668:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775555.900670:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491b800 x1862022710348288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775580 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900676:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775555.900678:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491b800 x1862022710348288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775580 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900683:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775555.900684:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775555.900686:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775555.900689:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775555.900691:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775555.900692:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307110924 00000100:00000001:0.0:1775775555.900694:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307114759 00000100:00000001:0.0:1775775555.900696:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307110924 00000100:00000001:0.0:1775775555.900697:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307114759 00000100:00000001:0.0:1775775555.900699:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775555.900700:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775555.900701:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775555.900703:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775555/real 1775775555] req@ffff8f623491b800 x1862022710348288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775555.900713:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775555.900714:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775555.900715:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775555.900717:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775555.900717:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775555.900719:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775555.900720:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491b800 x1862022710348288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775555.900725:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491b800 x1862022710348288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900729:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775555.900730:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775555.900731:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775555.900732:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775555.900734:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491b800 x1862022710348288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900738:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775555.900740:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491b800 x1862022710348288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900744:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775555.900746:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775555.900747:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775555.900748:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775555.900749:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775555.900752:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775555.900755:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775555.900757:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775555.900758:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775555.900761:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491b800 x1862022710348288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775555.900766:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710348288:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775555.900769:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775555.900771:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918a80 x1862022710348416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900775:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775555.900776:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775555.900778:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775555.900779:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775555.900780:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775555.900782:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710348416:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775555.900783:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775555.900785:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775555.900786:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775555.900787:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775555.900789:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775555.900790:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775555.900792:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775555.900793:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775555.900795:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775555.900797:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710348416, portal 10 00000100:00000001:0.0:1775775555.900798:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775555.900799:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684361344 : -123823025190272 : ffff8f6234918a80) 00000100:00000040:0.0:1775775555.900803:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918a80 x1862022710348416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775580 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900807:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775555.900809:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775555.900810:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775555.900811:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710348416, offset 0 00000400:00000200:0.0:1775775555.900814:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775555.900817:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775555.900819:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775555.900820:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775555.900822:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775555.900824:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775555.900825:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775555.900827:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453274:453274:256:4294967295] 192.168.201.115@tcp LPNI seq info [453199:453199:8:4294967295] 00000400:00000200:0.0:1775775555.900830:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775555.900834:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775555.900836:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775555.900838:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775555.900840:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775555.900842:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775775555.900844:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775555.900846:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775775555.900849:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775555.900850:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775555.900852:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775555.900854:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775555.900856:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775555.900857:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775555.900858:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775555.900860:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918a80 x1862022710348416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775580 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900865:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775555.900867:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918a80 x1862022710348416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775580 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900871:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775555.900872:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775555.900874:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775555.900876:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775555.900877:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775555.900879:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307110924 00000100:00000001:0.0:1775775555.900881:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307114759 00000100:00000001:0.0:1775775555.900882:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307110924 00000100:00000001:0.0:1775775555.900884:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307114759 00000100:00000001:0.0:1775775555.900885:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775555.900886:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775555.900887:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775555.900889:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775555/real 1775775555] req@ffff8f6234918a80 x1862022710348416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775555.900910:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775555.900911:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775555.900912:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775555.900914:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775555.900915:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775555.900917:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775555.900919:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918a80 x1862022710348416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775555.900925:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918a80 x1862022710348416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900929:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775555.900930:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775555.900931:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775555.900932:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775555.900934:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918a80 x1862022710348416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900938:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775555.900940:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918a80 x1862022710348416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900946:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775555.900949:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775555.900951:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775555.900952:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775555.900954:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775555.900956:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775555.900958:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775555.900962:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775555.900964:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775555.900968:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918a80 x1862022710348416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775555.900976:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710348416:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775555.900980:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775555.900982:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775555.900985:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491b800 x1862022710348288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.900993:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775555.900995:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775555.900997:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775775555.900999:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775555.901001:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775555.901002:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775555.901005:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775555.901007:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4d000. 02000000:00000001:0.0:1775775555.901009:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775555.901010:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775555.901011:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491b800. 00000100:00000001:0.0:1775775555.901013:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775555.901014:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775555.901015:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775555.901019:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918a80 x1862022710348416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775580 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775555.901032:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775555.901033:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775555.901035:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775555.901037:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775555.901038:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775555.901040:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775555.901041:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775555.901043:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:0.0:1775775555.901045:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775555.901047:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775555.901048:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918a80. 00000100:00000001:0.0:1775775555.901050:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775555.901052:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775555.901054:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775555.901058:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775555.901059:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775555.901061:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775555.901063:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775556.027251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775556.027257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775556.027263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775556.027264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775556.027267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775556.027268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775556.027338:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775556.027344:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775556.027348:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775556.923219:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775556.923227:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775556.923230:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775556.923237:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775556.923242:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775556.923247:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775556.923250:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775556.923252:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775556.923255:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775557.051290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775557.051296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775557.051302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775557.051303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775557.051306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775557.051307:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775557.947201:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775557.947206:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775557.947208:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775557.947212:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775557.947215:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775557.947218:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775557.947220:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775557.947222:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775557.947224:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775558.075213:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775558.075218:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775558.075222:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775558.075225:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775558.075227:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775558.075229:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775558.075232:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775558.075233:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775558.075235:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775558.075236:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775558.075239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775558.075240:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775558.075241:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775558.075242:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775558.075243:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775558.075244:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775558.075245:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775558.075247:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775558.075298:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775558.075302:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775558.075305:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775559.099212:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775559.099217:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775559.099222:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775559.099223:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775559.099226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775559.099227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775559.803242:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307114759, at 4307114816 00000100:00000010:0.0:1775775559.803252:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00010000:00000001:3.0:1775775560.123206:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775560.123212:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775560.123215:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775560.123216:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775560.123220:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775560.123221:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775560.123236:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775560.123238:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775560.123240:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775561.019271:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775561.019275:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775561.019277:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775561.019283:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775561.019286:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775561.019289:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775561.019293:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775561.019295:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775561.019299:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775561.019553:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775561.019561:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775561.019564:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775561.019579:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491aa00 x1862022710348544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.019605:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775561.019608:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775561.019613:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775561.019619:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775561.019624:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775561.019631:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710348544:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775561.019637:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775561.019644:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775561.019648:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775561.019655:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775561.019664:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775775561.019672:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775561.019682:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775561.019690:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775561.019700:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775561.019711:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710348544, portal 25 00000100:00000001:0.0:1775775561.019715:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775561.019719:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684369408 : -123823025182208 : ffff8f623491aa00) 00000100:00000040:0.0:1775775561.019734:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491aa00 x1862022710348544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.019752:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775561.019758:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775561.019765:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775561.019770:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710348544, offset 0 00000400:00000200:0.0:1775775561.019789:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775561.019819:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775561.019832:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775561.019841:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775561.019849:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775561.019854:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775561.019861:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775561.019873:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453275:453275:256:4294967295] 192.168.201.115@tcp LPNI seq info [453200:453200:8:4294967295] 00000400:00000200:0.0:1775775561.019884:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775561.019903:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775561.019919:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775561.019926:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775561.019942:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775561.019949:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775775561.019965:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775561.019971:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775775561.019980:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775561.019984:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775561.019991:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775561.020000:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775561.020007:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775561.020010:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775561.020017:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775561.020023:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491aa00 x1862022710348544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020038:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775561.020042:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491aa00 x1862022710348544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775586 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020048:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775561.020051:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775561.020058:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775561.020073:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775561.020078:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307119878 00000100:00000010:0.0:1775775561.020089:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775561.020099:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775561.020109:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307119878 00000100:00000010:0.0:1775775561.020114:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c34e0. 00000100:00000001:0.0:1775775561.020119:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775561.020126:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775561.020130:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775561.020180:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775561.020186:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775561/real 1775775561] req@ffff8f623491aa00 x1862022710348544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775561.020195:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775561.020198:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775561.020201:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775561.020204:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775561.020207:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775561.020211:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775561.020215:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491aa00 x1862022710348544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775561.020222:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491aa00 x1862022710348544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020226:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775561.020227:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775561.020229:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775561.020231:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775561.020233:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491aa00 x1862022710348544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020238:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775561.020243:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491aa00 x1862022710348544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020251:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775561.020254:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775561.020256:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775561.020257:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775561.020261:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775561.020263:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775561.020266:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775561.020272:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775561.020275:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775561.020280:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491aa00 x1862022710348544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775561.020289:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710348544:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775561.020298:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775561.020301:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775561.020305:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491aa00 x1862022710348544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020313:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775561.020316:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775561.020320:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775775561.020323:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775561.020325:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775561.020326:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775561.020328:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775561.020330:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:0.0:1775775561.020334:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775561.020335:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775561.020337:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491aa00. 00000100:00000001:0.0:1775775561.020339:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775561.020340:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775561.020342:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775561.020349:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775561.020350:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775561.020352:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775561.020354:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775561.020355:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775561.020358:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918e00 x1862022710348672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020364:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775561.020366:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775561.020367:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775561.020369:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775561.020371:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775561.020372:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710348672:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775561.020374:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775561.020376:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775561.020377:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775561.020379:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775561.020381:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775775561.020382:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775561.020386:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775561.020388:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775561.020391:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775561.020393:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710348672, portal 10 00000100:00000001:0.0:1775775561.020395:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775561.020396:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684362240 : -123823025189376 : ffff8f6234918e00) 00000100:00000040:0.0:1775775561.020401:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918e00 x1862022710348672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020406:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775561.020407:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775561.020409:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775561.020410:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710348672, offset 0 00000400:00000200:0.0:1775775561.020413:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775561.020416:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775561.020418:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775561.020420:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775561.020422:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775561.020424:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775561.020425:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775561.020428:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453276:453276:256:4294967295] 192.168.201.115@tcp LPNI seq info [453201:453201:8:4294967295] 00000400:00000200:0.0:1775775561.020431:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775561.020436:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775561.020438:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775561.020440:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775561.020443:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775561.020445:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775775561.020449:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775561.020451:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775775561.020454:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775561.020455:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775561.020457:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775561.020460:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775561.020462:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775561.020463:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775561.020465:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775561.020467:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918e00 x1862022710348672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020473:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775561.020475:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918e00 x1862022710348672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775586 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020479:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775561.020480:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775561.020482:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775561.020485:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775561.020487:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775561.020488:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307116044 00000100:00000001:0.0:1775775561.020490:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307119878 00000100:00000001:0.0:1775775561.020492:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307116044 00000100:00000001:0.0:1775775561.020493:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307119878 00000100:00000001:0.0:1775775561.020495:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775561.020497:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775561.020498:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775561.020500:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775561/real 1775775561] req@ffff8f6234918e00 x1862022710348672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775561.020505:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775561.020506:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775561.020506:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775561.020508:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775561.020509:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775561.020510:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775561.020512:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918e00 x1862022710348672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775561.020518:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918e00 x1862022710348672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020523:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775561.020523:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775561.020524:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775561.020525:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775561.020527:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918e00 x1862022710348672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020531:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775561.020534:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918e00 x1862022710348672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020538:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775561.020539:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775561.020540:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775561.020541:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775561.020542:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775561.020547:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775561.020548:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775561.020550:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775561.020552:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775561.020554:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918e00 x1862022710348672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775561.020558:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710348672:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775561.020561:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775561.020563:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918380 x1862022710348800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020572:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775561.020573:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775561.020575:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775561.020576:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775561.020578:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775561.020579:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710348800:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775561.020581:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775561.020582:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775561.020583:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775561.020584:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775561.020586:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775561.020587:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775561.020589:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775561.020590:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775561.020592:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775561.020593:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710348800, portal 10 00000100:00000001:0.0:1775775561.020595:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775561.020596:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684359552 : -123823025192064 : ffff8f6234918380) 00000100:00000040:0.0:1775775561.020599:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918380 x1862022710348800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020604:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775561.020605:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775561.020607:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775561.020608:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710348800, offset 0 00000400:00000200:0.0:1775775561.020611:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775561.020613:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775561.020615:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775561.020616:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775561.020618:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775561.020619:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775561.020621:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775561.020623:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453277:453277:256:4294967295] 192.168.201.115@tcp LPNI seq info [453202:453202:8:4294967295] 00000400:00000200:0.0:1775775561.020626:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775561.020630:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775561.020633:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775561.020635:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775561.020637:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775561.020638:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775775561.020641:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775561.020642:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775775561.020645:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775561.020646:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775561.020648:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775561.020650:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775561.020652:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775561.020653:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775561.020655:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775561.020657:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918380 x1862022710348800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020662:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775561.020664:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918380 x1862022710348800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775586 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020668:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775561.020669:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775561.020670:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775561.020672:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775561.020674:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775561.020675:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307116044 00000100:00000001:0.0:1775775561.020676:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307119878 00000100:00000001:0.0:1775775561.020678:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307116044 00000100:00000001:0.0:1775775561.020679:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307119878 00000100:00000001:0.0:1775775561.020681:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775561.020683:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775561.020684:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775561.020686:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775561/real 1775775561] req@ffff8f6234918380 x1862022710348800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775561.020692:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775561.020693:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775561.020693:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775561.020695:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775561.020696:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775561.020697:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775561.020699:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918380 x1862022710348800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775561.020705:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918380 x1862022710348800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020709:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775561.020709:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775561.020710:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775561.020711:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775561.020713:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918380 x1862022710348800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020717:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775561.020719:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918380 x1862022710348800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020723:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775561.020724:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775561.020725:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775561.020726:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775561.020727:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775561.020728:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775561.020730:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775561.020732:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775561.020734:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775561.020736:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918380 x1862022710348800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775561.020741:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710348800:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775561.020743:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775561.020744:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775561.020746:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918e00 x1862022710348672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020750:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775561.020751:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775561.020752:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775775561.020753:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775561.020754:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775561.020754:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775561.020756:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775561.020756:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:0.0:1775775561.020758:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775561.020759:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775561.020760:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918e00. 00000100:00000001:0.0:1775775561.020761:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775561.020762:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775561.020763:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775561.020765:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918380 x1862022710348800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775586 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775561.020769:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775561.020770:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775561.020771:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775561.020772:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775561.020773:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775561.020774:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775561.020775:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775561.020777:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4c000. 02000000:00000001:0.0:1775775561.020779:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775561.020781:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775561.020782:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918380. 00000100:00000001:0.0:1775775561.020784:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775561.020785:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775561.020786:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775561.020788:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775561.020790:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775561.020793:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775561.020795:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775561.147262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775561.147268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775561.147272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775561.147273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775561.147276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775561.147277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775561.147349:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775561.147353:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775561.147355:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775562.043234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775562.043241:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775562.043258:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775562.043262:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775562.043265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775562.043269:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775562.043271:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775562.043273:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775562.043276:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775562.171253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775562.171258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775562.171262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775562.171263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775562.171266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775562.171267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775563.067217:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775563.067225:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775563.067226:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775563.067232:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775563.067236:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775563.067239:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775563.067241:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775563.067244:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775563.067246:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775563.195278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775563.195287:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775563.195289:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775563.195293:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775563.195298:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775563.195301:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775563.195303:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775563.195304:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775563.195308:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775563.195309:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775563.195312:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775563.195313:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775563.195379:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775563.195384:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775563.195389:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775564.091254:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775564.091261:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775564.091263:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775564.091269:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775564.091273:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775564.091277:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775564.091280:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775564.091282:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775564.091285:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775564.219246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775564.219251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775564.219255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775564.219256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775564.219259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775564.219260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775564.923226:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307119878, at 4307119936 00000100:00000010:0.0:1775775564.923233:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00000100:00000001:2.0:1775775565.115206:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775565.115212:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775565.115214:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775565.115218:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775565.115220:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775565.115223:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775565.115226:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775565.115228:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775565.115230:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775565.243235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775565.243240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775565.243244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775565.243245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775565.243248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775565.243248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775565.243268:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775565.243270:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775565.243272:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775566.139226:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775566.139234:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775566.139238:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775566.139245:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775566.139252:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775566.139258:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775566.139263:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775566.139267:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775566.139273:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775566.139425:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775566.139429:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775566.139432:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775566.139442:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234919f80 x1862022710348928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.139455:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775566.139458:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775566.139464:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775566.139467:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775566.139478:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775566.139484:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234919f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710348928:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775566.139490:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775566.139494:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775566.139496:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775566.139501:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775566.139508:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775566.139512:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775566.139517:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775566.139521:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775566.139530:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775566.139539:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710348928, portal 25 00000100:00000001:0.0:1775775566.139547:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775566.139550:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684366720 : -123823025184896 : ffff8f6234919f80) 00000100:00000040:0.0:1775775566.139557:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234919f80 x1862022710348928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775591 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.139570:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775566.139575:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775566.139579:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775566.139584:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710348928, offset 0 00000400:00000200:0.0:1775775566.139594:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775566.139606:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775566.139613:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775566.139622:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775566.139629:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775566.139637:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775566.139640:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775566.139644:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453278:453278:256:4294967295] 192.168.201.115@tcp LPNI seq info [453203:453203:8:4294967295] 00000400:00000200:0.0:1775775566.139649:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775566.139660:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775566.139670:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775566.139679:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775566.139691:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775566.139699:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775775566.139713:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775566.139744:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775775566.139757:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775566.139763:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775566.139772:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775566.139783:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775566.139792:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775566.139797:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775566.139804:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775566.139811:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234919f80 x1862022710348928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775591 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.139843:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775566.139848:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234919f80 x1862022710348928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775591 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.139863:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775566.139867:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775566.139874:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775566.139883:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775566.139886:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307124998 00000100:00000010:0.0:1775775566.139893:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775566.139903:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775566.139910:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307124998 00000100:00000010:0.0:1775775566.139915:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c37e0. 00000100:00000001:0.0:1775775566.139919:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775566.139922:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775566.139925:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775566.139927:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775566.139933:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775566/real 1775775566] req@ffff8f6234919f80 x1862022710348928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775566.139944:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775566.139946:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775566.139949:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775566.139952:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775566.139954:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775566.139957:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775566.139961:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234919f80 x1862022710348928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775566.139969:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234919f80 x1862022710348928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.139975:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775566.139976:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775566.139978:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775566.139980:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775566.139983:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234919f80 x1862022710348928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.139987:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775566.139990:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234919f80 x1862022710348928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.139995:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775566.139997:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775566.139998:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775566.139999:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775566.140000:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775566.140002:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775566.140004:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775566.140006:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775566.140008:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775566.140010:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234919f80 x1862022710348928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775566.140015:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234919f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710348928:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775566.140018:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775566.140020:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775566.140022:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234919f80 x1862022710348928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.140026:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775566.140028:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775566.140030:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775566.140032:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775566.140033:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775566.140035:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775566.140036:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775566.140038:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4f400. 02000000:00000001:0.0:1775775566.140041:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775566.140042:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775566.140043:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234919f80. 00000100:00000001:0.0:1775775566.140046:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775566.140047:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775566.140048:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775566.140054:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775566.140055:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775566.140056:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775566.140057:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775566.140058:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775566.140060:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491a680 x1862022710349056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.140065:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775566.140066:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775566.140068:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775566.140069:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775566.140071:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775566.140072:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710349056:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775566.140074:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775566.140076:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775566.140077:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775566.140078:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775566.140080:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775566.140082:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775566.140085:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775566.140087:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775566.140089:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775566.140091:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710349056, portal 10 00000100:00000001:0.0:1775775566.140093:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775566.140094:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684368512 : -123823025183104 : ffff8f623491a680) 00000100:00000040:0.0:1775775566.140098:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491a680 x1862022710349056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775591 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.140103:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775566.140104:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775566.140106:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775566.140107:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710349056, offset 0 00000400:00000200:0.0:1775775566.140110:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775566.140113:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775566.140115:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775566.140117:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775566.140119:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775566.140121:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775566.140122:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775566.140125:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453279:453279:256:4294967295] 192.168.201.115@tcp LPNI seq info [453204:453204:8:4294967295] 00000400:00000200:0.0:1775775566.140128:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775566.140132:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775566.140165:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775566.140168:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775566.140171:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775566.140173:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775775566.140176:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775566.140178:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775775566.140181:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775566.140182:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775566.140184:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775566.140187:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775566.140189:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775566.140190:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775566.140192:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775566.140194:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491a680 x1862022710349056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775591 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.140199:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775566.140201:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491a680 x1862022710349056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775591 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.140210:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775566.140211:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775566.140213:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775566.140216:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775566.140218:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775566.140220:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307121164 00000100:00000001:0.0:1775775566.140222:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307124998 00000100:00000001:0.0:1775775566.140224:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307121164 00000100:00000001:0.0:1775775566.140225:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307124998 00000100:00000001:0.0:1775775566.140226:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775566.140228:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775566.140229:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775566.140231:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775566/real 1775775566] req@ffff8f623491a680 x1862022710349056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775566.140236:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775566.140237:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775566.140238:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775566.140239:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775566.140240:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775566.140241:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775566.140243:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491a680 x1862022710349056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775566.140248:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491a680 x1862022710349056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.140252:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775566.140253:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775566.140254:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775566.140255:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775566.140257:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491a680 x1862022710349056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.140262:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775566.140265:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491a680 x1862022710349056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.140269:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775566.140271:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775566.140271:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775566.140272:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775566.140273:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775566.140277:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775566.140278:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775566.140280:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775566.140281:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775566.140283:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491a680 x1862022710349056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775566.140288:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710349056:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775566.140291:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775566.140293:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491a300 x1862022710349184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.140297:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775566.140298:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775566.140300:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775566.140301:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775566.140302:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775566.140304:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710349184:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775566.140305:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775566.140306:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775566.140308:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775566.140309:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775566.140312:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775775566.140313:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775566.140315:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775566.140317:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775566.140319:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775566.140320:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710349184, portal 10 00000100:00000001:0.0:1775775566.140322:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775566.140323:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684367616 : -123823025184000 : ffff8f623491a300) 00000100:00000040:0.0:1775775566.140326:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491a300 x1862022710349184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775591 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.140331:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775566.140332:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775566.140333:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775566.140335:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710349184, offset 0 00000400:00000200:0.0:1775775566.140338:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775566.140340:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775566.140342:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775566.140344:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775566.140346:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775566.140347:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775566.140349:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775566.140351:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453280:453280:256:4294967295] 192.168.201.115@tcp LPNI seq info [453205:453205:8:4294967295] 00000400:00000200:0.0:1775775566.140354:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775566.140357:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775566.140360:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775566.140362:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775566.140365:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775566.140367:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775775566.140369:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775566.140372:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775775566.140374:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775566.140376:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775566.140378:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775566.140380:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775566.140382:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775566.140383:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775566.140384:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775566.140386:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491a300 x1862022710349184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775591 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.140391:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775566.140392:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491a300 x1862022710349184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775591 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.140397:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775566.140398:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775566.140399:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775566.140401:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775566.140403:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775566.140404:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307121164 00000100:00000001:0.0:1775775566.140405:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307124998 00000100:00000001:0.0:1775775566.140406:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307121164 00000100:00000001:0.0:1775775566.140408:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307124998 00000100:00000001:0.0:1775775566.140409:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775566.140411:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775566.140412:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775566.140414:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775566/real 1775775566] req@ffff8f623491a300 x1862022710349184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775566.140419:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775566.140420:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775566.140420:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775566.140422:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775566.140423:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775566.140425:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775566.140427:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491a300 x1862022710349184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775566.140432:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491a300 x1862022710349184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.140436:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775566.140437:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775566.140438:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775566.140438:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775566.140440:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491a300 x1862022710349184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.140444:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775566.140446:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491a300 x1862022710349184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.140450:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775566.140452:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775566.140453:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775566.140453:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775566.140454:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775566.140456:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775566.140457:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775566.140459:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775566.140460:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775566.140462:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491a300 x1862022710349184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775566.140467:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710349184:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775566.140469:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775566.140471:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775566.140472:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491a680 x1862022710349056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.140477:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775566.140478:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775566.140479:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775566.140480:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775566.140481:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775566.140482:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775566.140483:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775566.140484:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:0.0:1775775566.140486:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775566.140487:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775566.140488:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491a680. 00000100:00000001:0.0:1775775566.140489:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775566.140490:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775566.140491:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775566.140493:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491a300 x1862022710349184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775591 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775566.140497:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775566.140498:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775566.140499:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775775566.140500:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775566.140501:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775566.140502:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775566.140503:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775566.140504:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4d400. 02000000:00000001:0.0:1775775566.140505:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775566.140506:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775566.140506:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491a300. 00000100:00000001:0.0:1775775566.140508:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775566.140508:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775566.140509:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775566.140511:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775566.140512:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775566.140513:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775566.140515:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775566.267220:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775566.267225:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775566.267229:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775566.267230:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775566.267233:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775566.267234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775566.267285:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775566.267287:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775566.267298:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775567.163209:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775567.163215:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775567.163216:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775567.163222:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775567.163224:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775567.163227:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775567.163230:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775567.163232:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775567.163234:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775567.291842:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775567.291848:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775567.291853:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775567.291856:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775567.291859:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775567.291862:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775567.291865:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775567.291868:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775567.291870:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775567.291873:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775567.291877:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775567.291879:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775567.291880:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775567.291882:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775567.291884:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775567.291885:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775567.291886:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775567.291888:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775775567.995245:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:0.0:1775775567.995252:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:0.0:1775775567.995257:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1775775567.995434:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775775567.995457:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775775567.995462:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775775567.995465:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775568.187231:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775568.187238:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775568.187239:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775568.187246:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775568.187250:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775568.187253:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775568.187256:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775568.187259:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775568.187261:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775568.315252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775568.315258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775568.315264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775568.315265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775568.315269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775568.315271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775568.315335:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775568.315340:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775568.315345:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775569.339224:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775569.339230:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775569.339235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775569.339237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775569.339240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775569.339241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775570.043281:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307124998, at 4307125056 00000100:00000010:0.0:1775775570.043301:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00010000:00000001:3.0:1775775570.363217:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775570.363222:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775570.363226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775570.363227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775570.363230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775570.363231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775570.363248:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775570.363250:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775570.363252:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775571.259238:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775571.259250:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775571.259254:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775571.259263:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775571.259270:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775571.259281:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12453 00000400:00000200:0.0:1775775571.259293:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775571.259302:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775571.259306:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775571.259309:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775571.259313:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775571.259315:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775571.259318:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775571.259322:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775571.259325:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775571.259328:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775571.259331:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775571.259334:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775571.259336:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775571.259340:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775571.259342:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775571.259344:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775571.259346:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775571.259350:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775571.259353:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775571.259355:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775571.259359:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775571.259363:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775571.259367:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775571.259369:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775571.259371:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775571.259384:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062d80. 00000020:00000040:0.0:1775775571.259389:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775571.259395:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775571.259397:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775571.259401:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775571.259405:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775571.259410:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775571.259421:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6400. 00000100:00000001:0.0:1775775571.259430:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775571.259442:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062d80 x1862022710349312/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775571.259453:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775571.259458:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775571.259462:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009965f80b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062d80 x1862022710349312/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775571.259483:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775571.259485:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775571.259488:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775571.259489:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775571.259491:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775571.259494:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775571.259496:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12453 00000400:00000200:0.0:1775775571.259500:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775571.259502:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775571.259504:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775571.259507:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775571.259510:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775571.259511:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775571.259513:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775571.259515:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775571.259518:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775571.259519:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775571.259523:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775571.259526:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775571.259528:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775571.259533:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775571.259537:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775571.259541:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775571.259545:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775571.259550:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775571.259553:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775571.259558:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775571.259561:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775571.259569:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775571.259573:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775571.259578:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775571.259583:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775571.259587:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775571.259589:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775571.259594:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062a00. 00000020:00000040:0.0:1775775571.259596:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775571.259598:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775571.259599:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775571.259600:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775571.259603:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775571.259606:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775571.259613:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6c00. 00000100:00000001:0.0:1775775571.259616:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775571.259621:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062a00 x1862022710349440/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775571.259647:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775571.259648:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775571.259651:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009753f9f6] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062a00 x1862022710349440/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775571.259661:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775571.259663:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775571.259668:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775571.259671:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775571.259672:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775571.259674:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775571.259676:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12453 00000400:00000200:0.0:1775775571.259682:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775571.259685:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775571.259690:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775571.259692:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775571.259696:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775571.259701:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775571.259706:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775571.259710:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775571.259713:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775571.259716:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775571.259719:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775571.259723:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775571.259725:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775571.259730:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775571.259734:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775571.259735:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775571.259736:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775571.259738:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775571.259743:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775571.259748:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775571.259749:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775571.259751:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775571.259752:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775571.259754:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775571.259755:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775571.259760:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775571.259764:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775571.259767:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061500. 00000020:00000040:0.0:1775775571.259769:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775571.259773:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775571.259775:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775571.259778:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775571.259781:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775571.259782:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775571.259785:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5800. 00000100:00000001:0.0:1775775571.259790:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775571.259795:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061500 x1862022710349568/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775571.259808:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775571.259811:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775571.259818:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb8d992] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061500 x1862022710349568/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775571.259829:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775571.259831:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775571.259835:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775571.259841:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775571.259848:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775571.259855:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12463) 00000800:00000200:1.1:1775775571.259877:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775571.259880:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775775571.260198:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775571.260200:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775775571.260438:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775571.260439:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00010000:00000001:3.0:1775775571.387232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775571.387237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775571.387241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775571.387242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775571.387245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775571.387246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775571.387320:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775571.387323:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775571.387326:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775572.411216:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775572.411224:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775572.411226:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775572.411230:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775572.411235:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775572.411238:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775572.411240:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775572.411243:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775572.411247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775572.411249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775572.411252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775572.411253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775573.435268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775573.435274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775573.435284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775573.435285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775573.435289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775573.435290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775573.435371:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775573.435394:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775573.435401:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775574.459266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775574.459273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775574.459279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775574.459281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775574.459285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775574.459287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775575.163257:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307130118, at 4307130176 00000100:00000010:0.0:1775775575.163275:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2b40. 00010000:00000001:3.0:1775775575.483217:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775575.483222:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775575.483226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775575.483227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775575.483230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775575.483231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775575.483249:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775575.483251:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775575.483253:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775576.379289:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775576.379300:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775576.379303:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775576.379307:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775576.379312:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775576.379320:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12458 00000400:00000200:0.0:1775775576.379329:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775576.379336:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775576.379340:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775576.379344:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775576.379347:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775576.379349:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775576.379352:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775576.379356:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775576.379358:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775576.379362:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775576.379365:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775576.379367:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775576.379369:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775576.379372:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775576.379374:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775576.379376:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775576.379378:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775576.379381:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775576.379384:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775576.379387:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775576.379390:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775576.379394:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775576.379397:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775576.379399:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775576.379402:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775576.379413:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063b80. 00000020:00000040:0.0:1775775576.379417:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775576.379421:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775576.379423:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775576.379425:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775576.379428:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775576.379432:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775576.379441:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7000. 00000100:00000001:0.0:1775775576.379448:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775576.379457:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063b80 x1862022710349696/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775576.379466:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775576.379470:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775576.379474:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ee9007b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063b80 x1862022710349696/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775576.379493:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775576.379496:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775576.379500:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775576.379501:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775576.379502:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775576.379505:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775576.379507:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12458 00000400:00000200:0.0:1775775576.379510:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775576.379513:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775576.379515:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775576.379519:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775576.379522:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775576.379524:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775576.379530:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775576.379560:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775576.379562:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775576.379571:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775576.379577:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775576.379587:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775576.379597:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775576.379611:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775576.379615:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775576.379618:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775576.379627:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775576.379639:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775576.379648:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775576.379655:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775576.379661:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775576.379669:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775576.379676:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775576.379682:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775576.379687:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775576.379693:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775576.379697:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775576.379701:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061f80. 00000020:00000040:0.0:1775775576.379709:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775576.379713:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775576.379721:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775576.379723:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775576.379724:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775576.379727:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775576.379740:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7c00. 00000100:00000001:0.0:1775775576.379745:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775576.379754:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061f80 x1862022710349824/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775576.379785:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775576.379786:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775576.379795:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000029d3794] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061f80 x1862022710349824/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775576.379810:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775576.379813:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775576.379824:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775576.379830:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775576.379835:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775576.379839:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775576.379846:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12458 00000400:00000200:0.0:1775775576.379864:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775576.379868:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775576.379870:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775576.379875:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775576.379879:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775576.379882:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775576.379887:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775576.379895:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775576.379897:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775576.379902:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775576.379905:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775576.379907:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775576.379912:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775576.379921:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775576.379927:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775576.379932:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775576.379937:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775576.379943:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775576.379951:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775576.379958:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775576.379968:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775576.379971:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775576.379983:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775576.379985:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775576.379987:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775576.379991:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775576.380001:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775576.380006:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060700. 00000020:00000040:0.0:1775775576.380008:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775576.380010:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775576.380015:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775576.380019:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775576.380022:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775576.380024:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775576.380034:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4800. 00000100:00000001:0.0:1775775576.380037:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775576.380044:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060700 x1862022710349952/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775576.380078:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.1:1775775576.380082:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775775576.380083:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775576.380092:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000100:00000040:0.0:1775775576.380100:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000225c8695] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060700 x1862022710349952/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775576.380126:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775576.380130:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775576.380155:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775576.380157:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775576.380160:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775576.380164:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12469) 00000800:00000200:1.1:1775775576.380565:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775576.380566:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775775576.380765:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775576.380767:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00010000:00000001:3.0:1775775576.507273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775576.507280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775576.507285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775576.507287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775576.507300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775576.507302:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775576.507363:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775576.507366:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775576.507369:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775577.531216:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775577.531222:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775577.531226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775577.531229:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775577.531230:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775577.531233:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775577.531236:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775577.531238:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775577.531239:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775577.531241:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775577.531243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775577.531244:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775577.531245:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775577.531246:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775577.531247:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775577.531249:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775577.531250:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775577.531252:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775578.555257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775578.555262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775578.555266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775578.555267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775578.555270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775578.555271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775578.555348:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775578.555353:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775578.555357:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775579.579248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775579.579254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775579.579258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775579.579260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775579.579264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775579.579265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775580.283273:0:214512:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307135239, at 4307135296 00000100:00000010:0.0:1775775580.283288:0:214512:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2660. 00010000:00000001:3.0:1775775580.603277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775580.603285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775580.603290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775580.603291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775580.603294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775580.603296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775580.603323:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775580.603325:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775580.603328:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775581.499291:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775581.499306:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775581.499312:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775581.499318:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775581.499326:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775581.499336:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12464 00000400:00000200:2.0:1775775581.499350:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775581.499358:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775581.499363:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775581.499367:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775581.499371:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775581.499373:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775581.499376:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775581.499380:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775581.499383:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775581.499386:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775581.499389:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775581.499392:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775581.499396:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775581.499399:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775581.499402:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775581.499404:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775581.499406:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775581.499410:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775581.499413:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775581.499416:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775581.499420:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775581.499424:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775581.499428:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775581.499431:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775581.499433:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775581.499445:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775775581.499472:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775581.499477:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775581.499479:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775581.499483:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775581.499486:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775581.499491:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775581.499500:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775775581.499508:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775581.499519:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710350080/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775581.499530:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775581.499534:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775581.499538:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710350080/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775581.499564:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775581.499566:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775581.499570:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775581.499571:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775581.499573:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775581.499575:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775581.499577:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12464 00000400:00000200:2.0:1775775581.499581:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775581.499584:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775581.499586:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775581.499589:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775581.499591:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775581.499593:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775581.499595:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775581.499597:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775581.499599:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775581.499601:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775581.499604:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775581.499607:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775581.499609:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775581.499615:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775581.499618:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775581.499622:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775581.499626:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775581.499629:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775581.499634:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775581.499637:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775581.499640:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775581.499648:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775581.499652:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775581.499656:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775581.499660:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775581.499665:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775581.499669:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775581.499676:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775775581.499679:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775581.499680:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775581.499682:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775581.499686:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775581.499688:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775581.499692:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775581.499699:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775775581.499703:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775581.499713:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710350208/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775581.499728:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775581.499732:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775581.499738:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710350208/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775581.499755:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775581.499761:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775581.499772:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775581.499776:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775581.499780:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775581.499787:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775581.499792:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12464 00000400:00000200:2.0:1775775581.499800:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775581.499806:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775581.499811:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775581.499817:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775581.499821:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775581.499826:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775581.499829:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775581.499833:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775581.499836:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775581.499838:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775581.499841:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775581.499845:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775581.499850:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775581.499858:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775581.499863:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775581.499868:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775581.499873:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775581.499875:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775581.499878:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775581.499881:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775581.499885:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775581.499890:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775581.499894:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775581.499898:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775581.499902:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775581.499905:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775581.499909:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775581.499914:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775775581.499918:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000800:00000200:1.1:1775775581.499923:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:2.0:1775775581.499924:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775581.499927:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775581.499930:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775581.499932:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775581.499934:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775581.499939:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775775581.499942:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775581.499948:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710350336/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775581.499966:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775581.499970:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775581.499975:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710350336/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775581.499984:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775581.499986:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775581.499991:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775581.499995:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775581.500000:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775581.500005:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12474) 00000800:00000200:1.1:1775775581.500128:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775581.500299:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00010000:00000001:3.0:1775775581.627286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775581.627293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775581.627299:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775581.627300:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775581.627303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775581.627304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775581.627372:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775581.627376:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775581.627380:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775582.651249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775582.651255:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775582.651257:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775582.651259:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775582.651263:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775582.651265:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775582.651267:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775582.651268:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775582.651271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775582.651272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775582.651275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775582.651276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775583.675243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775583.675249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775583.675253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775583.675254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775583.675258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775583.675260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775583.675382:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775583.675388:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775583.675392:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775775584.571416:0:2400:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:3.0:1775775584.571452:0:2400:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1775775584.571462:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775584.571466:0:2400:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:3.0:1775775584.571472:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775584.571475:0:2400:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:3.0:1775775584.571481:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775775584.571484:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:3.0:1775775584.571490:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775775584.571494:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775775584.571500:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3b28 00000400:00000010:3.0:1775775584.571503:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3b28. 00000100:00000001:3.0:1775775584.571510:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775775584.571520:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f623344a680 x1862022710350080/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775584.571545:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775775584.571550:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344a680 x1862022710350080/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775606 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775584.571574:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775584.571579:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775775584.571600:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775775584.571613:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230a00. 00000400:00000200:3.0:1775775584.571620:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775775584.571628:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775775584.571654:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3088 00000400:00000010:3.0:1775775584.571666:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3088. 00000100:00000001:3.0:1775775584.571674:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775775584.571687:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6233449f80 x1862022710350208/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775584.571727:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775775584.571738:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449f80 x1862022710350208/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775606 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775584.571763:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775584.571768:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775775584.571777:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775775584.571781:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230700. 00000400:00000200:3.0:1775775584.571791:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775775584.571799:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775775584.571802:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3770 00000400:00000010:3.0:1775775584.571807:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3770. 00000100:00000001:3.0:1775775584.571817:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775775584.571828:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6233448380 x1862022710350336/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775584.571885:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775775584.571901:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448380 x1862022710350336/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775606 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775584.571922:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775584.571929:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:3.0:1775775584.699257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775584.699263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775584.699268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775584.699269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775584.699273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775584.699274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775585.403370:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307140358, at 4307140416 00000100:00000010:0.0:1775775585.403405:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2de0. 00010000:00000001:3.0:1775775585.723239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775585.723245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775585.723250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775585.723251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775585.723255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775585.723256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775585.723276:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775585.723279:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775585.723281:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775586.619278:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775586.619288:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775586.619291:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775586.619298:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775586.619303:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775586.619309:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775586.619313:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775586.619320:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775586.619325:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775586.619667:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775586.619679:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775586.619688:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775586.619709:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234919c00 x1862022710350464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.619744:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775586.619750:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775586.619758:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775586.619767:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775586.619783:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775586.619801:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234919c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710350464:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775586.619817:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775586.619826:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775586.619833:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775586.619848:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775586.619864:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775775586.619874:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775586.619897:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775586.619909:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775586.619926:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775586.619945:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710350464, portal 25 00000100:00000001:0.0:1775775586.619953:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775586.619962:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684365824 : -123823025185792 : ffff8f6234919c00) 00000100:00000040:0.0:1775775586.619981:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234919c00 x1862022710350464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775611 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.620026:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775586.620040:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775586.620049:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775586.620059:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710350464, offset 0 00000400:00000200:0.0:1775775586.620076:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775586.620103:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775586.620124:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775586.620216:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775586.620228:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775586.620240:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775586.620248:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775586.620264:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453290:453290:256:4294967295] 192.168.201.115@tcp LPNI seq info [453215:453215:8:4294967295] 00000400:00000200:0.0:1775775586.620286:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775586.620337:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775586.620390:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775586.620404:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775586.620427:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775586.620439:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775775586.620453:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775586.620463:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775775586.620471:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775586.620475:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775586.620486:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775586.620495:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775586.620503:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775586.620506:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775586.620512:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775586.620520:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234919c00 x1862022710350464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775611 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.620535:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775586.620541:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234919c00 x1862022710350464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775611 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.620551:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775586.620554:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775586.620562:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775586.620569:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775586.620573:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307145479 00000100:00000010:0.0:1775775586.620581:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775586.620591:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775586.620599:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307145479 00000100:00000010:0.0:1775775586.620605:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c38a0. 00000100:00000001:0.0:1775775586.620609:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775586.620614:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775586.620617:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775586.620619:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775586.620625:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775586/real 1775775586] req@ffff8f6234919c00 x1862022710350464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775586.620644:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775586.620647:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775586.620650:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775586.620654:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775586.620657:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775586.620661:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775586.620666:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234919c00 x1862022710350464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775586.620679:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234919c00 x1862022710350464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.620691:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775586.620693:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775586.620697:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775586.620700:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775586.620706:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234919c00 x1862022710350464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.620733:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775586.620739:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234919c00 x1862022710350464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.620750:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775586.620754:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775586.620756:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775586.620757:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775586.620761:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775586.620764:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775586.620769:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775586.620774:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775586.620777:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775586.620782:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234919c00 x1862022710350464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775586.620791:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234919c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710350464:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775586.620796:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775586.620798:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775586.620802:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234919c00 x1862022710350464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.620811:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775586.620814:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775586.620818:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775775586.620822:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775586.620825:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775586.620827:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775586.620830:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775586.620832:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4f000. 02000000:00000001:0.0:1775775586.620837:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775586.620840:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775586.620843:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234919c00. 00000100:00000001:0.0:1775775586.620847:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775586.620849:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775586.620851:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775586.620876:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775586.620879:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775586.620883:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775586.620885:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775586.620887:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775586.620894:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491a300 x1862022710350592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.620905:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775586.620907:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775586.620911:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775586.620914:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775586.620917:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775586.620921:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710350592:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775586.620926:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775586.620930:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775586.620932:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775586.620934:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775586.620939:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775775586.620942:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775586.620948:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775586.620952:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775586.620957:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775586.620962:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710350592, portal 10 00000100:00000001:0.0:1775775586.620965:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775586.620967:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684367616 : -123823025184000 : ffff8f623491a300) 00000100:00000040:0.0:1775775586.620974:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491a300 x1862022710350592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775611 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.620984:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775586.620987:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775586.620991:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775586.620994:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710350592, offset 0 00000400:00000200:0.0:1775775586.621000:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775586.621006:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775586.621012:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775586.621016:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775586.621020:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775586.621023:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775586.621026:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775586.621030:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453291:453291:256:4294967295] 192.168.201.115@tcp LPNI seq info [453216:453216:8:4294967295] 00000400:00000200:0.0:1775775586.621035:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775586.621044:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775586.621051:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775586.621055:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775586.621060:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775586.621064:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775775586.621071:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775586.621074:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775775586.621080:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775586.621082:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775586.621087:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775586.621091:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775586.621096:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775586.621098:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775586.621101:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775586.621115:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491a300 x1862022710350592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775611 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.621127:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775586.621131:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491a300 x1862022710350592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775611 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.621173:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775586.621175:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775586.621180:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775586.621186:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775586.621190:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775586.621193:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307141645 00000100:00000001:0.0:1775775586.621197:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307145479 00000100:00000001:0.0:1775775586.621200:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307141645 00000100:00000001:0.0:1775775586.621203:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307145479 00000100:00000001:0.0:1775775586.621206:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775586.621209:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775586.621211:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775586.621216:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775586/real 1775775586] req@ffff8f623491a300 x1862022710350592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775586.621227:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775586.621229:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775586.621232:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775586.621236:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775586.621238:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775586.621241:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775586.621245:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491a300 x1862022710350592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775586.621256:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491a300 x1862022710350592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.621281:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775586.621283:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775586.621285:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775586.621287:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775586.621291:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491a300 x1862022710350592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.621299:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775586.621304:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491a300 x1862022710350592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.621313:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775586.621317:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775586.621319:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775586.621320:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775586.621322:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775586.621329:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775586.621332:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775586.621337:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775586.621340:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775586.621345:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491a300 x1862022710350592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775586.621356:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710350592:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775586.621377:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775586.621386:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491a680 x1862022710350720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.621395:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775586.621396:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775586.621399:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775586.621401:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775586.621404:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775586.621408:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710350720:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775586.621412:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775586.621415:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775586.621417:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775586.621420:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775586.621425:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7c00. 02000000:00000001:0.0:1775775586.621427:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775586.621431:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775586.621434:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775586.621438:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775586.621442:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710350720, portal 10 00000100:00000001:0.0:1775775586.621444:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775586.621445:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684368512 : -123823025183104 : ffff8f623491a680) 00000100:00000040:0.0:1775775586.621453:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491a680 x1862022710350720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775611 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.621460:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775586.621463:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775586.621465:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775586.621468:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710350720, offset 0 00000400:00000200:0.0:1775775586.621472:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775586.621477:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775586.621481:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775586.621485:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775586.621489:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775586.621491:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775586.621494:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775586.621497:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453292:453292:256:4294967295] 192.168.201.115@tcp LPNI seq info [453217:453217:8:4294967295] 00000400:00000200:0.0:1775775586.621502:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775586.621509:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775586.621514:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775586.621527:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775586.621533:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775586.621536:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775775586.621541:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775586.621544:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775775586.621548:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775586.621550:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775586.621554:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775586.621559:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775586.621562:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775586.621564:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775586.621567:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775586.621571:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491a680 x1862022710350720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775611 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.621581:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775586.621585:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491a680 x1862022710350720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775611 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.621594:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775586.621596:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775586.621600:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775586.621603:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775586.621607:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775586.621610:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307141645 00000100:00000001:0.0:1775775586.621613:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307145479 00000100:00000001:0.0:1775775586.621616:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307141645 00000100:00000001:0.0:1775775586.621619:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307145479 00000100:00000001:0.0:1775775586.621622:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775586.621624:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775586.621626:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775586.621631:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775586/real 1775775586] req@ffff8f623491a680 x1862022710350720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775586.621640:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775586.621642:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775586.621644:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775586.621647:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775586.621649:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775586.621652:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775586.621662:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491a680 x1862022710350720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775586.621672:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491a680 x1862022710350720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.621679:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775586.621681:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775586.621683:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775586.621684:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775586.621689:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491a680 x1862022710350720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.621697:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775586.621701:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491a680 x1862022710350720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.621710:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775586.621713:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775586.621715:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775586.621716:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775586.621719:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775586.621721:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775586.621723:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775586.621728:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775586.621731:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775586.621736:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491a680 x1862022710350720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775586.621747:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710350720:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775586.621750:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775586.621752:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775586.621756:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491a300 x1862022710350592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.621762:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775586.621763:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775586.621765:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775775586.621768:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775586.621769:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775586.621770:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775586.621772:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775586.621774:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4d400. 02000000:00000001:0.0:1775775586.621776:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775586.621777:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775586.621778:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491a300. 00000100:00000001:0.0:1775775586.621781:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775586.621781:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775586.621783:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775586.621785:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491a680 x1862022710350720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775611 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775586.621790:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775586.621791:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775586.621793:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7c00. 02000000:00000001:0.0:1775775586.621796:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775586.621798:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775586.621799:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775586.621802:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775586.621804:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4fc00. 02000000:00000001:0.0:1775775586.621806:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775586.621807:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775586.621809:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491a680. 00000100:00000001:0.0:1775775586.621812:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775586.621813:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775586.621815:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775586.621821:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775586.621823:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775586.621825:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775586.621827:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775586.747311:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775586.747317:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775586.747323:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775586.747324:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775586.747328:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775586.747329:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775586.747433:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775586.747436:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775586.747440:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775587.643245:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775587.643251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775587.643252:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775587.643257:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775587.643260:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775587.643263:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775587.643265:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775587.643267:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775587.643269:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775587.771207:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775587.771212:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775587.771216:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775587.771219:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775587.771221:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775587.771223:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775587.771225:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775587.771227:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775587.771228:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775587.771230:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775587.771232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775587.771233:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775587.771234:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775587.771235:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775587.771236:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775587.771237:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775587.771238:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775587.771239:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775588.667313:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775588.667370:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775588.667373:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775588.667380:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775588.667385:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775588.667390:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775588.667394:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775588.667397:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775588.667401:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775588.795245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775588.795252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775588.795258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775588.795259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775588.795263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775588.795264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775588.795316:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775588.795335:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775588.795339:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775589.819355:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775589.819363:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775589.819369:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775589.819371:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775589.819375:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775589.819377:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775590.523248:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307145479, at 4307145536 00000100:00000010:0.0:1775775590.523272:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00010000:00000001:3.0:1775775590.843280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775590.843328:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775590.843336:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775590.843338:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775590.843342:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775590.843344:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775590.843373:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775590.843376:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775590.843379:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775591.739283:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775591.739292:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775591.739294:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775591.739306:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775591.739311:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775591.739317:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775591.739324:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775591.739328:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775591.739333:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775591.739508:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775591.739513:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775591.739517:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775591.739528:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234919f80 x1862022710350848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.739548:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775591.739550:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775591.739556:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775591.739558:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775591.739562:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775591.739568:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234919f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710350848:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775591.739575:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775591.739579:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775591.739582:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775591.739587:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775591.739594:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7c00. 02000000:00000001:0.0:1775775591.739597:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775591.739604:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775591.739610:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775591.739617:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775591.739625:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710350848, portal 25 00000100:00000001:0.0:1775775591.739630:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775591.739632:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684366720 : -123823025184896 : ffff8f6234919f80) 00000100:00000040:0.0:1775775591.739641:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234919f80 x1862022710350848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775616 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.739656:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775591.739662:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775591.739666:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775591.739671:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710350848, offset 0 00000400:00000200:0.0:1775775591.739683:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775591.739693:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775591.739701:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775591.739709:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775591.739716:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775591.739723:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775591.739729:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775591.739736:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453293:453293:256:4294967295] 192.168.201.115@tcp LPNI seq info [453218:453218:8:4294967295] 00000400:00000200:0.0:1775775591.739744:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775591.739759:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775591.739768:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775591.739775:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775591.739784:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775591.739790:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775775591.739802:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775591.739808:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1775775591.739818:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775591.739822:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775591.739831:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775591.739840:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775591.739844:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775591.739848:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775591.739852:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775591.739855:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234919f80 x1862022710350848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775616 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.739863:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775591.739866:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234919f80 x1862022710350848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775616 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.739872:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775591.739874:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775591.739879:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775591.739883:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775591.739886:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307150598 00000100:00000010:0.0:1775775591.739891:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775591.739898:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775591.739904:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307150598 00000100:00000010:0.0:1775775591.739907:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3a20. 00000100:00000001:0.0:1775775591.739911:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775591.739924:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775591.739927:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775591.739930:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775591.739935:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775591/real 1775775591] req@ffff8f6234919f80 x1862022710350848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775591.739945:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775591.739947:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775591.739950:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775591.739953:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775591.739954:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775591.739957:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775591.739960:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234919f80 x1862022710350848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775591.739971:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234919f80 x1862022710350848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.739976:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775591.739977:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775591.739979:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775591.739980:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775591.739982:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234919f80 x1862022710350848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.739986:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775591.739989:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234919f80 x1862022710350848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.739994:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775591.739996:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775591.739997:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775591.739998:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775591.740000:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775591.740002:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775591.740004:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775591.740006:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775591.740008:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775591.740010:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234919f80 x1862022710350848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775591.740016:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234919f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710350848:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775591.740019:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775591.740021:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775591.740023:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234919f80 x1862022710350848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.740031:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775591.740033:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775591.740035:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7c00. 02000000:00000001:0.0:1775775591.740037:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775591.740038:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775591.740039:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775591.740041:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775591.740043:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:0.0:1775775591.740046:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775591.740047:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775591.740048:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234919f80. 00000100:00000001:0.0:1775775591.740050:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775591.740051:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775591.740053:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775591.740059:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775591.740060:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775591.740061:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775591.740063:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775591.740064:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775591.740066:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918380 x1862022710350976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.740071:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775591.740072:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775591.740073:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775591.740075:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775591.740076:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775591.740078:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710350976:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775591.740080:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775591.740081:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775591.740082:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775591.740083:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775591.740085:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7c00. 02000000:00000001:0.0:1775775591.740087:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775591.740089:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775591.740091:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775591.740094:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775591.740096:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710350976, portal 10 00000100:00000001:0.0:1775775591.740097:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775591.740098:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684359552 : -123823025192064 : ffff8f6234918380) 00000100:00000040:0.0:1775775591.740102:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918380 x1862022710350976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775616 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.740107:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775591.740108:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775591.740110:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775591.740111:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710350976, offset 0 00000400:00000200:0.0:1775775591.740114:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775591.740116:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775591.740119:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775591.740121:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775591.740123:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775591.740124:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775591.740126:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775591.740128:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453294:453294:256:4294967295] 192.168.201.115@tcp LPNI seq info [453219:453219:8:4294967295] 00000400:00000200:0.0:1775775591.740131:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775591.740159:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775591.740162:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775591.740165:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775591.740167:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775591.740169:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775775591.740172:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775591.740174:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1775775591.740176:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775591.740177:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775591.740179:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775591.740182:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775591.740184:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775591.740186:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775591.740188:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775591.740191:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918380 x1862022710350976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775616 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.740196:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775591.740198:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918380 x1862022710350976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775616 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.740203:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775591.740204:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775591.740206:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775591.740209:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775591.740212:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775591.740213:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307146764 00000100:00000001:0.0:1775775591.740215:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307150598 00000100:00000001:0.0:1775775591.740217:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307146764 00000100:00000001:0.0:1775775591.740218:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307150598 00000100:00000001:0.0:1775775591.740220:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775591.740221:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775591.740222:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775591.740224:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775591/real 1775775591] req@ffff8f6234918380 x1862022710350976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775591.740229:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775591.740230:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775591.740231:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775591.740233:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775591.740234:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775591.740236:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775591.740238:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918380 x1862022710350976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775591.740243:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918380 x1862022710350976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.740251:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775591.740252:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775591.740253:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775591.740254:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775591.740256:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918380 x1862022710350976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.740260:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775591.740262:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918380 x1862022710350976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.740266:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775591.740282:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775591.740283:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775591.740283:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775591.740285:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775591.740288:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775591.740289:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775591.740292:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775591.740293:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775591.740296:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918380 x1862022710350976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775591.740300:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710350976:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775591.740303:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775591.740305:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918e00 x1862022710351104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.740309:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775591.740310:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775591.740312:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775591.740313:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775591.740314:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775591.740316:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710351104:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775591.740318:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775591.740319:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775591.740320:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775591.740322:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775591.740324:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775775591.740325:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775591.740327:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775591.740329:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775591.740331:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775591.740333:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710351104, portal 10 00000100:00000001:0.0:1775775591.740335:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775591.740336:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684362240 : -123823025189376 : ffff8f6234918e00) 00000100:00000040:0.0:1775775591.740340:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918e00 x1862022710351104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775616 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.740344:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775591.740345:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775591.740347:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775591.740348:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710351104, offset 0 00000400:00000200:0.0:1775775591.740351:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775591.740353:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775591.740356:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775591.740357:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775591.740359:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775591.740361:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775591.740362:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775591.740365:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453295:453295:256:4294967295] 192.168.201.115@tcp LPNI seq info [453220:453220:8:4294967295] 00000400:00000200:0.0:1775775591.740367:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775591.740372:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775591.740375:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775591.740378:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775591.740380:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775591.740382:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775775591.740385:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775591.740386:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1775775591.740389:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775591.740390:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775591.740392:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775591.740394:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775591.740396:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775591.740397:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775591.740399:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775591.740401:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918e00 x1862022710351104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775616 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.740406:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775591.740408:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918e00 x1862022710351104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775616 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.740412:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775591.740413:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775591.740414:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775591.740416:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775591.740418:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775591.740419:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307146764 00000100:00000001:0.0:1775775591.740421:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307150598 00000100:00000001:0.0:1775775591.740422:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307146764 00000100:00000001:0.0:1775775591.740424:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307150598 00000100:00000001:0.0:1775775591.740425:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775591.740426:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775591.740428:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775591.740429:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775591/real 1775775591] req@ffff8f6234918e00 x1862022710351104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775591.740435:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775591.740436:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775591.740437:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775591.740438:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775591.740439:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775591.740440:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775591.740442:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918e00 x1862022710351104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775591.740447:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918e00 x1862022710351104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.740451:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775591.740451:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775591.740452:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775591.740453:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775591.740455:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918e00 x1862022710351104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.740459:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775591.740461:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918e00 x1862022710351104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.740465:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775591.740467:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775591.740468:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775591.740469:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775591.740470:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775591.740471:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775591.740473:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775591.740475:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775591.740476:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775591.740479:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918e00 x1862022710351104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775591.740485:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710351104:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775591.740487:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775591.740488:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775591.740490:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918380 x1862022710350976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.740493:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775591.740494:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775591.740495:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7c00. 02000000:00000001:0.0:1775775591.740497:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775591.740497:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775591.740498:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775591.740499:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775591.740500:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4f400. 02000000:00000001:0.0:1775775591.740502:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775591.740503:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775591.740504:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918380. 00000100:00000001:0.0:1775775591.740505:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775591.740506:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775591.740508:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775591.740509:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918e00 x1862022710351104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775616 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775591.740514:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775591.740514:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775591.740515:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775775591.740516:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775591.740517:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775591.740518:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775591.740519:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775591.740519:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:0.0:1775775591.740521:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775591.740522:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775591.740523:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918e00. 00000100:00000001:0.0:1775775591.740524:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775591.740525:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775591.740526:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775591.740528:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775591.740529:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775591.740531:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775591.740532:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775591.867353:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775591.867359:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775591.867365:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775591.867366:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775591.867369:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775591.867370:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775591.867430:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775591.867434:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775591.867438:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775592.763298:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775592.763307:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775592.763310:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775592.763317:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775592.763322:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775592.763328:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775592.763331:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775592.763333:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775592.763337:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775592.891278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775592.891287:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775592.891289:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775592.891292:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775592.891298:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775592.891300:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775592.891302:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775592.891303:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775592.891307:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775592.891308:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775592.891311:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775592.891312:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775593.787267:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775593.787277:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775593.787279:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775593.787286:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775593.787291:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775593.787295:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775593.787298:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775593.787301:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775593.787303:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775593.915326:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775593.915334:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775593.915343:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775593.915344:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775593.915349:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775593.915358:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775593.915451:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775593.915457:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775593.915460:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775594.811294:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775594.811301:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775594.811302:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775594.811309:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775594.811313:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775594.811317:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775594.811320:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775594.811322:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775594.811325:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775594.939285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775594.939291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775594.939298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775594.939299:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775594.939303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775594.939304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775595.643249:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307150598, at 4307150656 00000100:00000010:0.0:1775775595.643263:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00000100:00000001:2.0:1775775595.835243:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775595.835249:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775595.835251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775595.835255:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775595.835258:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775595.835261:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775595.835263:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775595.835265:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775595.835267:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775595.963239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775595.963243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775595.963247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775595.963248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775595.963252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775595.963253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775595.963271:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775595.963273:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775595.963275:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775596.859264:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775596.859272:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775596.859274:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775596.859280:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775596.859284:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775596.859288:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775596.859291:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775596.859294:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775596.859298:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775596.859599:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775596.859604:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775596.859609:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775596.859625:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491aa00 x1862022710351232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.859638:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775596.859641:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775596.859646:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775596.859649:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775596.859650:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775596.859659:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710351232:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775596.859671:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775596.859677:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775596.859680:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775596.859685:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775596.859694:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775775596.859697:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775596.859713:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775596.859720:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775596.859731:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775596.859739:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710351232, portal 25 00000100:00000001:0.0:1775775596.859752:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775596.859753:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684369408 : -123823025182208 : ffff8f623491aa00) 00000100:00000040:0.0:1775775596.859766:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491aa00 x1862022710351232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775621 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.859783:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775596.859786:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775596.859789:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775596.859796:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710351232, offset 0 00000400:00000200:0.0:1775775596.859803:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775596.859817:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775596.859825:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775596.859837:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775596.859846:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775596.859853:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775596.859863:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775596.859872:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453296:453296:256:4294967295] 192.168.201.115@tcp LPNI seq info [453221:453221:8:4294967295] 00000400:00000200:0.0:1775775596.859883:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775596.859898:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775596.859904:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775596.859916:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775596.859931:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775596.859936:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775775596.859952:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775596.859961:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1775775596.859973:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775596.859978:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775596.859989:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775596.860004:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775596.860014:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775596.860020:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775596.860029:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775596.860038:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491aa00 x1862022710351232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775621 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.860086:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775596.860091:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491aa00 x1862022710351232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775621 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.860108:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775596.860116:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775596.860125:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775596.860166:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775596.860170:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307155719 00000100:00000010:0.0:1775775596.860177:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775596.860199:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775596.860210:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307155719 00000100:00000010:0.0:1775775596.860218:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c30c0. 00000100:00000001:0.0:1775775596.860227:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775596.860234:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775596.860242:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775596.860251:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775596.860265:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775596/real 1775775596] req@ffff8f623491aa00 x1862022710351232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775596.860287:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775596.860291:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775596.860296:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775596.860302:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775596.860307:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775596.860313:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775596.860321:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491aa00 x1862022710351232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775596.860355:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491aa00 x1862022710351232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.860374:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775596.860378:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775596.860385:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775596.860389:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775596.860401:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491aa00 x1862022710351232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.860451:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775596.860464:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491aa00 x1862022710351232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.860496:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775596.860502:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775596.860511:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775596.860512:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775596.860516:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775596.860518:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775596.860522:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775596.860528:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775596.860530:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775596.860535:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491aa00 x1862022710351232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775596.860545:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710351232:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775596.860550:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775596.860552:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775596.860555:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491aa00 x1862022710351232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.860561:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775596.860563:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775596.860566:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775775596.860569:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775596.860571:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775596.860574:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775596.860576:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775596.860579:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:0.0:1775775596.860582:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775596.860584:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775596.860587:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491aa00. 00000100:00000001:0.0:1775775596.860590:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775596.860592:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775596.860594:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775596.860600:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775596.860602:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775596.860605:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775596.860606:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775596.860608:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775596.860612:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918a80 x1862022710351360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.860621:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775596.860622:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775596.860625:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775596.860627:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775596.860630:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775596.860633:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710351360:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775596.860636:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775596.860638:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775596.860640:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775596.860642:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775596.860646:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775775596.860648:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775596.860652:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775596.860655:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775596.860659:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775596.860662:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710351360, portal 10 00000100:00000001:0.0:1775775596.860664:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775596.860666:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684361344 : -123823025190272 : ffff8f6234918a80) 00000100:00000040:0.0:1775775596.860672:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918a80 x1862022710351360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775621 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.860679:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775596.860682:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775596.860684:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775596.860687:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710351360, offset 0 00000400:00000200:0.0:1775775596.860691:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775596.860695:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775596.860700:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775596.860703:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775596.860706:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775596.860709:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775596.860711:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775596.860714:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453297:453297:256:4294967295] 192.168.201.115@tcp LPNI seq info [453222:453222:8:4294967295] 00000400:00000200:0.0:1775775596.860719:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775596.860725:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775596.860731:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775596.860734:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775596.860738:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775596.860741:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775775596.860747:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775596.860749:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1775775596.860753:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775596.860755:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775596.860758:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775596.860761:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775596.860764:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775596.860766:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775596.860768:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775596.860772:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918a80 x1862022710351360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775621 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.860780:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775596.860783:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918a80 x1862022710351360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775621 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.860797:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775596.860799:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775596.860802:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775596.860806:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775596.860810:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775596.860811:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307151884 00000100:00000001:0.0:1775775596.860814:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307155719 00000100:00000001:0.0:1775775596.860817:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307151884 00000100:00000001:0.0:1775775596.860819:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307155719 00000100:00000001:0.0:1775775596.860821:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775596.860823:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775596.860825:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775596.860829:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775596/real 1775775596] req@ffff8f6234918a80 x1862022710351360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775596.860837:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775596.860839:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775596.860840:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775596.860842:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775596.860844:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775596.860846:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775596.860849:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918a80 x1862022710351360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775596.860858:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918a80 x1862022710351360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.860865:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775596.860866:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775596.860868:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775596.860870:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775596.860873:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918a80 x1862022710351360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.860880:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775596.860884:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918a80 x1862022710351360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.860891:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775596.860894:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775596.860895:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775596.860897:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775596.860898:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775596.860903:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775596.860906:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775596.860909:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775596.860911:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775596.860915:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918a80 x1862022710351360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775596.860923:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710351360:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775596.860927:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775596.860931:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491b800 x1862022710351488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.860938:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775596.860939:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775596.860942:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775596.860944:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775596.860946:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775596.860948:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710351488:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775596.860951:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775596.860954:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775596.860956:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775596.860958:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775596.860961:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7c00. 02000000:00000001:0.0:1775775596.860963:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775596.860967:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775596.860970:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775596.860972:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775596.860975:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710351488, portal 10 00000100:00000001:0.0:1775775596.860977:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775596.860978:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684372992 : -123823025178624 : ffff8f623491b800) 00000100:00000040:0.0:1775775596.860984:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491b800 x1862022710351488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775621 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.860990:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775596.860993:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775596.860995:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775596.860998:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710351488, offset 0 00000400:00000200:0.0:1775775596.861002:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775596.861005:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775596.861009:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775596.861012:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775596.861014:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775596.861017:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775596.861020:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775596.861023:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453298:453298:256:4294967295] 192.168.201.115@tcp LPNI seq info [453223:453223:8:4294967295] 00000400:00000200:0.0:1775775596.861027:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775596.861033:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775596.861037:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775596.861040:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775596.861044:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775596.861047:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775775596.861051:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775596.861053:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1775775596.861057:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775596.861060:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775596.861062:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775596.861066:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775596.861068:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775596.861070:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775596.861072:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775596.861075:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491b800 x1862022710351488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775621 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.861082:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775596.861085:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491b800 x1862022710351488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775621 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.861091:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775596.861093:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775596.861096:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775596.861098:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775596.861101:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775596.861103:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307151884 00000100:00000001:0.0:1775775596.861106:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307155719 00000100:00000001:0.0:1775775596.861108:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307151884 00000100:00000001:0.0:1775775596.861110:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307155719 00000100:00000001:0.0:1775775596.861112:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775596.861114:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775596.861116:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775596.861119:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775596/real 1775775596] req@ffff8f623491b800 x1862022710351488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775596.861127:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775596.861128:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775596.861129:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775596.861132:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775596.861156:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775596.861159:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775596.861162:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491b800 x1862022710351488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775596.861171:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491b800 x1862022710351488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.861177:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775596.861191:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775596.861193:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775596.861194:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775596.861198:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491b800 x1862022710351488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.861205:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775596.861208:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491b800 x1862022710351488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.861215:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775596.861217:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775596.861219:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775596.861220:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775596.861222:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775596.861224:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775596.861226:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775596.861230:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775596.861232:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775596.861236:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491b800 x1862022710351488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775596.861245:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710351488:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775596.861248:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775596.861251:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775596.861254:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918a80 x1862022710351360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.861262:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775596.861263:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775596.861265:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775775596.861268:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775596.861269:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775596.861271:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775596.861273:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775596.861274:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:0.0:1775775596.861277:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775596.861279:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775596.861280:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918a80. 00000100:00000001:0.0:1775775596.861283:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775596.861284:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775596.861286:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775596.861289:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491b800 x1862022710351488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775621 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775596.861296:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775596.861297:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775596.861298:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7c00. 02000000:00000001:0.0:1775775596.861301:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775596.861303:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775596.861304:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775596.861306:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775596.861308:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4c400. 02000000:00000001:0.0:1775775596.861310:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775596.861311:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775596.861313:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491b800. 00000100:00000001:0.0:1775775596.861315:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775596.861316:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775596.861318:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775596.861322:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775596.861323:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775596.861325:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775596.861327:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775596.987277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775596.987287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775596.987295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775596.987296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775596.987300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775596.987302:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775596.987377:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775596.987381:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775596.987385:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775597.883300:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775597.883310:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775597.883313:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775597.883322:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775597.883327:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775597.883333:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775597.883336:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775597.883339:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775597.883344:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775598.011303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775598.011309:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775598.011315:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775598.011319:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775598.011321:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775598.011324:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775598.011327:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775598.011330:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775598.011332:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775598.011333:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775598.011336:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775598.011337:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775598.011338:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775598.011339:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775598.011340:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775598.011340:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775598.011343:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775598.011344:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775598.907254:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775598.907260:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775598.907261:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775598.907265:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775598.907268:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775598.907271:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775598.907274:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775598.907276:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775598.907278:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775599.035263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775599.035267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775599.035271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775599.035272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775599.035275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775599.035276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775599.035336:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775599.035342:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775599.035346:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775600.059278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775600.059285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775600.059290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775600.059291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775600.059294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775600.059295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775600.763270:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307155719, at 4307155776 00000100:00000010:0.0:1775775600.763280:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00010000:00000001:3.0:1775775601.083280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775601.083287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775601.083293:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775601.083294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775601.083297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775601.083298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775601.083325:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775601.083327:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775601.083330:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775601.979227:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775601.979236:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775601.979241:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775601.979247:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775601.979255:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775601.979263:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12484 00000400:00000200:0.0:1775775601.979274:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775601.979279:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775601.979282:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775601.979285:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775601.979288:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775601.979290:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775601.979292:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775601.979296:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775601.979297:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775601.979300:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775601.979302:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775601.979304:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775601.979306:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775601.979308:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775601.979310:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775601.979311:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775601.979312:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775601.979314:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775601.979317:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775601.979320:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775601.979323:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775601.979326:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775601.979328:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775601.979330:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775601.979332:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775601.979341:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061880. 00000020:00000040:0.0:1775775601.979344:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775601.979348:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775601.979350:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775601.979352:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775601.979355:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775601.979360:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775601.979368:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4800. 00000100:00000001:0.0:1775775601.979374:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775601.979383:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061880 x1862022710351616/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775601.979392:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775601.979397:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775601.979400:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005afd1f27] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061880 x1862022710351616/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775601.979421:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775601.979424:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775601.979428:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775601.979430:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775601.979432:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775601.979436:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775601.979439:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12484 00000400:00000200:0.0:1775775601.979444:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775601.979450:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775601.979456:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775601.979463:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775601.979470:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775601.979475:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775601.979481:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775601.979484:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775601.979486:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775601.979490:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775601.979493:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775601.979497:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775601.979502:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775601.979507:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775601.979510:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775601.979513:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775601.979516:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775601.979518:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775601.979520:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775601.979522:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775601.979523:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775601.979531:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775601.979535:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775601.979538:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775601.979543:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775601.979547:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775601.979551:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775601.979555:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061180. 00000020:00000040:0.0:1775775601.979556:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775601.979558:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775601.979560:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775601.979563:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775601.979564:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775601.979568:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775601.979572:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7c00. 00000100:00000001:0.0:1775775601.979576:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775601.979584:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061180 x1862022710351744/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775601.979609:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775601.979611:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775601.979616:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e23859] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061180 x1862022710351744/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775601.979624:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775601.979628:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775601.979633:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775601.979635:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775601.979637:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775601.979639:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775601.979643:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12484 00000400:00000200:0.0:1775775601.979648:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775601.979650:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775601.979652:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775601.979654:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775601.979656:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775601.979660:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775601.979662:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775601.979664:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775601.979665:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775601.979666:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775601.979668:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775601.979670:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775601.979675:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775601.979678:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775601.979680:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775601.979684:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775601.979687:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775601.979689:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775601.979694:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775601.979696:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775601.979698:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775601.979701:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775601.979704:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775601.979707:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775601.979709:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775601.979712:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775601.979715:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775601.979719:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061c00. 00000020:00000040:0.0:1775775601.979724:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775601.979726:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775601.979729:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775601.979731:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775601.979732:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775601.979733:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775601.979736:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7000. 00000100:00000001:0.0:1775775601.979742:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775601.979749:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061c00 x1862022710351872/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775601.979760:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775601.979762:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775601.979766:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076a44879] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061c00 x1862022710351872/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775601.979772:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775601.979776:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775601.979781:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775601.979786:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775775601.979791:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:0.0:1775775601.979792:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775775601.979796:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000100:00000040:0.0:1775775601.979798:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12494) 00000800:00000200:1.1:1775775601.980122:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775601.980123:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775775601.980345:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775601.980347:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00010000:00000001:3.0:1775775602.107216:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775602.107226:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775602.107235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775602.107237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775602.107241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775602.107243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775602.107309:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775602.107313:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775602.107316:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775603.131218:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775603.131229:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775603.131232:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775603.131237:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775603.131242:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775603.131246:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775603.131249:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775603.131251:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775603.131257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775603.131259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775603.131263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775603.131265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775604.155264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775604.155269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775604.155273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775604.155274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775604.155277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775604.155278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775604.155344:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775604.155347:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775604.155350:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775605.179197:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775605.179202:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775605.179206:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775605.179207:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775605.179209:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775605.179218:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775775605.883291:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:0.0:1775775605.883299:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:0.0:1775775605.883305:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000100:00000040:0.0:1775775605.883352:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307160838, at 4307160896 00000100:00000010:0.0:1775775605.883363:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2300. 00000800:00000001:0.0:1775775605.883455:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775775605.883478:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775775605.883484:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775775605.883488:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1775775606.203257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775606.203262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775606.203266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775606.203267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775606.203270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775606.203271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775606.203292:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775606.203295:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775606.203297:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775607.099249:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775607.099262:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775607.099267:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775607.099273:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775607.099281:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775607.099291:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12489 00000400:00000200:2.0:1775775607.099303:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775607.099312:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775607.099316:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775607.099320:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775607.099323:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775607.099326:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775607.099328:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775607.099333:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775607.099335:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775607.099339:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775607.099342:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775607.099345:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775607.099347:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775607.099350:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775607.099354:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775607.099355:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775607.099358:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775607.099361:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775607.099365:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775607.099368:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775607.099372:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775607.099384:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775607.099387:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775607.099391:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775607.099393:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775607.099406:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775775607.099410:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775607.099414:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775607.099417:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775607.099421:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775607.099424:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775607.099430:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775607.099440:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775607.099449:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775607.099460:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710352000/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775607.099472:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775607.099477:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775607.099483:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710352000/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775607.099511:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775607.099514:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775607.099519:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775607.099521:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775607.099523:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775607.099527:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775607.099532:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12489 00000400:00000200:2.0:1775775607.099539:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775607.099545:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775607.099548:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775607.099551:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775607.099557:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775607.099562:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775607.099568:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775607.099574:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775607.099578:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775607.099581:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775607.099584:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775607.099590:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775607.099595:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775607.099601:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775607.099606:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775607.099611:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775607.099615:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775607.099618:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775607.099622:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775607.099626:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775607.099629:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775607.099636:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775607.099642:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775607.099644:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775607.099650:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775607.099654:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775607.099657:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775607.099663:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775775607.099667:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775607.099672:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775607.099676:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775607.099681:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775607.099684:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775607.099688:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775607.099695:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775775607.099700:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775607.099711:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710352128/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775607.099727:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775607.099731:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775607.099740:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710352128/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775607.099768:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775607.099774:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775607.099778:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775607.099780:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775607.099783:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775607.099791:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775607.099799:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12489 00000400:00000200:2.0:1775775607.099807:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775607.099814:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775607.099819:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775607.099828:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775607.099833:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1775775607.099837:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775775607.099840:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:1.1:1775775607.099842:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000100:00000001:2.0:1775775607.099844:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775607.099850:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775607.099854:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775607.099858:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775607.099861:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775607.099865:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775607.099868:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775607.099876:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775607.099882:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775607.099886:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775607.099891:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775607.099895:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775607.099900:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775607.099905:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775607.099910:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775607.099915:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775607.099919:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775607.099925:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775607.099928:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775607.099934:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775607.099938:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775607.099944:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775775607.099949:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775607.099955:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775607.099959:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775607.099961:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775607.099963:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775607.099967:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775607.099974:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775775607.099979:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775607.099987:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710352256/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775607.100020:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775607.100025:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775607.100034:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710352256/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775607.100067:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775607.100074:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775607.100081:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775607.100087:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775607.100094:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775607.100105:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12499) 00000800:00000200:1.1:1775775607.100444:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775607.100448:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775775607.100777:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775607.100781:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00010000:00000001:3.0:1775775607.227214:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775607.227219:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775607.227223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775607.227224:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775607.227227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775607.227228:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775607.227343:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775607.227350:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775607.227366:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775608.251307:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775608.251317:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775608.251325:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775608.251330:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775608.251333:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775608.251338:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775608.251343:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775608.251348:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775608.251352:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775608.251357:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775608.251382:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775608.251385:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775608.251387:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775608.251390:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775608.251393:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775608.251396:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775608.251399:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775608.251403:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775609.275275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775609.275284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775609.275292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775609.275295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775609.275301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775609.275304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775609.275443:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775609.275452:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775609.275458:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775610.299264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775610.299287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775610.299294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775610.299295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775610.299300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775610.299302:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775611.003285:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307165958, at 4307166016 00000100:00000010:0.0:1775775611.003299:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e25a0. 00010000:00000001:3.0:1775775611.323191:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775611.323195:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775611.323199:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775611.323200:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775611.323203:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775611.323204:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775611.323220:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775611.323222:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775611.323224:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775612.219259:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775612.219278:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775612.219283:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775612.219290:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775612.219301:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775612.219313:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12494 00000400:00000200:2.0:1775775612.219327:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775612.219335:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775612.219340:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775612.219343:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775612.219347:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775612.219350:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775612.219352:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775612.219358:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775612.219360:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775612.219364:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775612.219367:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775612.219369:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775612.219372:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775612.219376:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775612.219378:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775612.219381:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775612.219383:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775612.219387:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775612.219390:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775612.219394:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775612.219398:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775612.219402:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775612.219406:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775612.219409:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775612.219412:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775612.219424:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775775612.219429:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775612.219434:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775612.219436:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775612.219440:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775612.219444:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775612.219449:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775612.219460:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775775612.219468:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775612.219480:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710352384/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775612.219493:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775612.219499:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775612.219505:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710352384/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775612.219534:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775612.219537:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775612.219542:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775612.219543:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775612.219545:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775612.219548:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775612.219552:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12494 00000400:00000200:2.0:1775775612.219558:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775612.219563:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775612.219566:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775612.219570:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775612.219573:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775612.219575:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775612.219579:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775612.219584:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775612.219587:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775612.219591:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775612.219595:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775612.219599:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775612.219601:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775612.219610:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775612.219617:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775612.219622:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775612.219629:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775612.219638:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775612.219647:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775612.219652:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775612.219658:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775612.219668:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775612.219677:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775612.219685:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775612.219694:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775612.219702:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775612.219706:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775612.219713:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775775612.219716:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775612.219721:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775612.219726:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775612.219734:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775612.219737:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775612.219744:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775612.219752:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775612.219758:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775612.219772:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710352512/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775612.219811:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775612.219816:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775612.219828:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710352512/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775612.219857:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775612.219863:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775612.219866:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775612.219868:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775612.219874:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775612.219879:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775612.219886:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12494 00000400:00000200:2.0:1775775612.219898:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775612.219925:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775612.219933:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775612.219944:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775612.219952:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775612.219958:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775612.219967:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775612.219974:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775612.219979:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775612.219984:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775612.219988:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775612.219997:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775612.220005:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775612.220015:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775612.220025:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775612.220034:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775612.220040:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775612.220045:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000800:00000200:1.1:1775775612.220051:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00080000:2.0:1775775612.220055:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000800:00000200:1.1:1775775612.220060:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000100:00000001:2.0:1775775612.220061:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775612.220068:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775612.220073:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775612.220080:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775612.220086:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775612.220090:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775612.220096:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775612.220103:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775612.220111:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775775612.220116:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775612.220123:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775612.220127:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775612.220181:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775612.220185:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775612.220191:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775612.220206:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775775612.220213:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775612.220228:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710352640/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775612.220253:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775612.220260:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775612.220272:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710352640/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775612.220321:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775612.220335:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775612.220345:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775612.220349:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775612.220358:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775612.220369:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12504) 00000800:00000200:1.1:1775775612.220600:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775612.220603:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775775612.220792:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775612.220794:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00010000:00000001:3.0:1775775612.347195:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775612.347200:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775612.347202:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775612.347205:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775612.347208:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775612.347210:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775612.347212:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775612.347214:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775612.347217:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775612.347218:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775612.347221:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775612.347222:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775612.347273:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775612.347279:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775612.347282:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775613.371288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775613.371310:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775613.371315:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775613.371316:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775613.371319:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775613.371320:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775614.395247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775614.395256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775614.395264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775614.395265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775614.395270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775614.395271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775614.395393:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775614.395400:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775614.395407:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775615.419252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775615.419259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775615.419264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775615.419266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775615.419269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775615.419270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775616.123204:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307171079, at 4307171136 00000100:00000010:0.0:1775775616.123223:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2120. 00010000:00000001:3.0:1775775616.443245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775616.443252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775616.443258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775616.443259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775616.443262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775616.443264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775616.443287:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775616.443289:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775616.443292:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775617.339309:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775617.339341:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775617.339346:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775617.339354:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775617.339361:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775617.339375:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12499 00000400:00000200:3.0:1775775617.339409:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775617.339420:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775617.339424:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775617.339429:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775617.339433:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775617.339436:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775617.339439:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775617.339444:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775617.339462:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775617.339467:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775617.339471:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775617.339476:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775617.339480:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775617.339486:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775617.339490:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775617.339493:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775617.339496:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775617.339501:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775617.339507:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775617.339512:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775617.339518:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775617.339524:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775617.339530:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775617.339534:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775617.339538:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775617.339555:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775775617.339562:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775617.339585:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775617.339587:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775617.339592:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775617.339595:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775617.339603:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775617.339615:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000001:3.0:1775775617.339635:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775617.339648:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710352768/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775617.339680:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775617.339687:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775617.339692:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710352768/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775617.339722:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775617.339726:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775617.339730:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775617.339732:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775617.339734:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775617.339737:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775617.339741:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12499 00000400:00000200:3.0:1775775617.339748:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775617.339752:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775617.339755:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775617.339777:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775617.339779:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775617.339782:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775617.339788:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775617.339793:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775617.339796:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775617.339800:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775617.339803:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775617.339806:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775617.339810:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775617.339819:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775617.339844:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775617.339849:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775617.339857:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775617.339860:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775617.339867:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775617.339872:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775617.339877:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775617.339885:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775617.339891:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775617.339896:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775617.339904:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775617.339911:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775617.339914:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775617.339918:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775775617.339923:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775617.339929:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775617.339934:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775617.339935:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775617.339937:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775617.339939:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775617.339945:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000100:00000001:3.0:1775775617.339953:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775617.339962:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710352896/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775617.339980:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775617.339983:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775617.339991:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710352896/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775617.340001:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775617.340006:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775617.340011:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775617.340017:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775617.340022:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775617.340030:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775617.340037:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12499 00000400:00000200:3.0:1775775617.340053:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775617.340066:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775617.340075:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775617.340085:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775617.340095:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775617.340102:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775617.340108:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775617.340113:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775617.340114:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775617.340116:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775617.340119:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775617.340125:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775617.340131:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775617.340179:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775617.340182:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775617.340206:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775617.340210:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775617.340217:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775617.340229:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775617.340237:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775617.340243:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775617.340250:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775617.340257:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775617.340261:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775617.340264:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775617.340268:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775617.340274:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775617.340282:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775775617.340288:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775617.340295:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775617.340299:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775617.340305:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775617.340307:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775617.340310:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775617.340317:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000001:3.0:1775775617.340326:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775617.340335:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710353024/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775617.340348:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775617.340353:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775617.340379:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710353024/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775617.340398:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775617.340401:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775775617.340403:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775775617.340410:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000100:00000040:3.0:1775775617.340411:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775617.340419:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775617.340428:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775617.340438:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12510) 00000800:00000200:1.1:1775775617.341061:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775617.341063:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775775617.341506:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775617.341509:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00010000:00000001:3.0:1775775617.467309:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775617.467318:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775617.467326:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775617.467330:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775617.467333:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775617.467337:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775617.467340:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775617.467342:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775617.467344:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775617.467347:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775617.467351:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775617.467353:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775617.467354:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775617.467355:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775617.467357:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775617.467358:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775617.467360:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775617.467361:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775617.467485:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775617.467492:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775617.467498:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775618.491359:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775618.491370:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775618.491382:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775618.491385:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775618.491409:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775618.491411:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775619.515247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775619.515254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775619.515262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775619.515264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775619.515268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775619.515269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775619.515357:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775619.515361:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775619.515365:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775620.539206:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775620.539212:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775620.539218:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775620.539218:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775620.539221:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775620.539222:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775621.243252:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307176199, at 4307176256 00000100:00000010:0.0:1775775621.243296:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2f60. 00010000:00000001:3.0:1775775621.563288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775621.563298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775621.563307:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775621.563309:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775621.563314:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775621.563316:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775621.563369:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775621.563373:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775621.563376:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775775622.459506:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775622.459512:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775622.459516:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775622.459528:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918000 x1862022710353152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.459548:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775622.459551:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775622.459556:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775622.459562:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775622.459567:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775622.459574:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710353152:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775622.459582:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775622.459587:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775622.459590:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775622.459594:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775622.459603:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775775622.459609:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775622.459618:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775622.459624:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775622.459635:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775622.459647:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710353152, portal 25 00000100:00000001:0.0:1775775622.459652:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775622.459654:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684358656 : -123823025192960 : ffff8f6234918000) 00000100:00000040:0.0:1775775622.459663:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918000 x1862022710353152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.459678:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775622.459684:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775622.459688:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775622.459694:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710353152, offset 0 00000400:00000200:0.0:1775775622.459707:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775622.459721:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775622.459761:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775622.459772:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775622.459778:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775622.459785:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775622.459792:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775622.459803:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453311:453311:256:4294967295] 192.168.201.115@tcp LPNI seq info [453236:453236:8:4294967295] 00000400:00000200:0.0:1775775622.459819:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775622.459840:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775622.459850:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775622.459858:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775622.459868:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775622.459873:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775775622.459881:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775622.459885:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775775622.459890:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775622.459892:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775622.459899:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775622.459903:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775622.459909:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775622.459911:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775622.459916:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775622.459919:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918000 x1862022710353152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.459927:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775622.459930:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918000 x1862022710353152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775647 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.459935:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775622.459937:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775622.459942:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775622.459946:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775622.459948:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307181318 00000100:00000010:0.0:1775775622.459953:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775622.459959:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775622.459967:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307181318 00000100:00000010:0.0:1775775622.459970:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3720. 00000100:00000001:0.0:1775775622.459974:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775622.459977:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775622.459981:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775622.459984:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775622.459989:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775622/real 1775775622] req@ffff8f6234918000 x1862022710353152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775622.460001:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775622.460003:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775622.460006:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775622.460009:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775622.460011:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775622.460014:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775622.460018:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918000 x1862022710353152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775622.460027:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918000 x1862022710353152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.460032:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775622.460034:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775622.460036:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775622.460038:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775622.460041:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918000 x1862022710353152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.460046:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775622.460050:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918000 x1862022710353152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.460055:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775622.460057:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775622.460058:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775622.460058:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775622.460060:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775622.460062:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775622.460064:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775622.460074:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775622.460077:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775622.460080:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918000 x1862022710353152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775622.460086:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710353152:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775622.460089:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775622.460092:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775622.460095:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918000 x1862022710353152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.460102:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775622.460105:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775622.460108:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775775622.460111:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775622.460113:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775622.460114:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775622.460117:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775622.460120:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4dc00. 02000000:00000001:0.0:1775775622.460125:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775622.460127:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775622.460130:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918000. 00000100:00000001:0.0:1775775622.460169:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775622.460170:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775622.460173:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775622.460181:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775622.460183:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775622.460185:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775622.460186:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775622.460188:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775622.460193:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491b800 x1862022710353280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.460201:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775622.460202:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775622.460205:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775622.460207:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775622.460210:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775622.460213:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710353280:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775622.460217:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775622.460220:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775622.460222:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775622.460224:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775622.460227:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775775622.460229:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775622.460233:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775622.460235:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775622.460238:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775622.460240:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710353280, portal 10 00000100:00000001:0.0:1775775622.460241:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775622.460243:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684372992 : -123823025178624 : ffff8f623491b800) 00000100:00000040:0.0:1775775622.460248:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491b800 x1862022710353280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.460253:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775622.460254:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775622.460256:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775622.460257:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710353280, offset 0 00000400:00000200:0.0:1775775622.460260:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775622.460263:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775622.460265:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775622.460267:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775622.460269:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775622.460271:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775622.460273:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775622.460275:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453312:453312:256:4294967295] 192.168.201.115@tcp LPNI seq info [453237:453237:8:4294967295] 00000400:00000200:0.0:1775775622.460278:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775622.460282:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775622.460285:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775622.460288:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775622.460291:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775622.460293:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775775622.460297:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775622.460299:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775775622.460301:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775622.460303:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775622.460305:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775622.460307:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775622.460310:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775622.460311:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775622.460313:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775622.460317:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491b800 x1862022710353280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.460325:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775622.460328:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491b800 x1862022710353280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775647 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.460335:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775622.460337:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775622.460340:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775622.460344:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775622.460348:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775622.460350:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307177484 00000100:00000001:0.0:1775775622.460353:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307181318 00000100:00000001:0.0:1775775622.460356:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307177484 00000100:00000001:0.0:1775775622.460359:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307181318 00000100:00000001:0.0:1775775622.460361:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775622.460364:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775622.460365:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775622.460370:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775622/real 1775775622] req@ffff8f623491b800 x1862022710353280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775622.460386:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775622.460388:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775622.460390:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775622.460392:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775622.460394:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775622.460397:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775622.460400:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491b800 x1862022710353280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775622.460410:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491b800 x1862022710353280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.460418:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775622.460420:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775622.460421:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775622.460423:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775622.460427:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491b800 x1862022710353280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.460434:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775622.460439:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491b800 x1862022710353280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.460446:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775622.460448:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775622.460450:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775622.460452:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775622.460454:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775622.460460:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775622.460463:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775622.460467:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775622.460470:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775622.460475:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491b800 x1862022710353280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775622.460483:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710353280:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775622.460486:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775622.460488:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918a80 x1862022710353408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.460492:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775622.460494:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775622.460495:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775622.460497:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775622.460498:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775622.460500:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710353408:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775622.460502:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775622.460503:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775622.460505:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775622.460506:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775622.460510:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775622.460512:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775622.460516:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775622.460518:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775622.460522:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775622.460525:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710353408, portal 10 00000100:00000001:0.0:1775775622.460527:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775622.460528:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684361344 : -123823025190272 : ffff8f6234918a80) 00000100:00000040:0.0:1775775622.460535:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918a80 x1862022710353408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.460543:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775622.460545:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775622.460548:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775622.460551:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710353408, offset 0 00000400:00000200:0.0:1775775622.460556:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775622.460560:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775622.460564:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775622.460566:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775622.460568:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775622.460570:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775622.460572:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775622.460574:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453313:453313:256:4294967295] 192.168.201.115@tcp LPNI seq info [453238:453238:8:4294967295] 00000400:00000200:0.0:1775775622.460577:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775622.460582:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775622.460586:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775622.460588:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775622.460591:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775622.460594:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775775622.460598:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775622.460601:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775775622.460606:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775622.460608:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775622.460610:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775622.460614:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775622.460617:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775622.460619:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775622.460622:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775622.460626:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918a80 x1862022710353408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.460634:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775622.460638:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918a80 x1862022710353408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775647 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.460645:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775622.460646:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775622.460650:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775622.460652:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775622.460655:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775622.460657:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307177484 00000100:00000001:0.0:1775775622.460659:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307181318 00000100:00000001:0.0:1775775622.460660:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307177484 00000100:00000001:0.0:1775775622.460662:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307181318 00000100:00000001:0.0:1775775622.460663:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775622.460664:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775622.460666:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775622.460668:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775622/real 1775775622] req@ffff8f6234918a80 x1862022710353408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775622.460674:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775622.460675:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775622.460676:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775622.460678:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775622.460679:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775622.460680:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775622.460683:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918a80 x1862022710353408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775622.460688:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918a80 x1862022710353408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.460692:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775622.460693:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775622.460694:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775622.460695:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775622.460697:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918a80 x1862022710353408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.460702:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775622.460706:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918a80 x1862022710353408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.460712:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775622.460716:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775622.460717:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775622.460719:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775622.460721:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775622.460722:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775622.460725:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775622.460729:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775622.460744:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775622.460750:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918a80 x1862022710353408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775622.460759:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710353408:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775622.460764:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775622.460766:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775622.460769:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491b800 x1862022710353280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.460775:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775622.460776:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775622.460778:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775775622.460780:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775622.460781:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775622.460782:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775622.460784:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775622.460786:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:0.0:1775775622.460789:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775622.460790:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775622.460792:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491b800. 00000100:00000001:0.0:1775775622.460794:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775622.460796:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775622.460797:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775622.460801:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918a80 x1862022710353408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775647 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775622.460815:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775622.460816:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775622.460818:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775622.460820:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775622.460822:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775622.460823:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775622.460825:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775622.460827:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4c400. 02000000:00000001:0.0:1775775622.460829:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775622.460830:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775622.460832:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918a80. 00000100:00000001:0.0:1775775622.460834:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775622.460835:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775622.460837:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775622.460841:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775622.460843:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775622.460845:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775622.460847:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775622.587268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775622.587276:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775622.587280:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775622.587283:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775622.587289:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775622.587291:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775622.587293:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775622.587295:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775622.587298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775622.587299:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775622.587302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775622.587303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775622.587361:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775622.587365:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775622.587368:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775623.611285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775623.611293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775623.611300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775623.611301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775623.611306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775623.611308:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775624.635233:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775624.635241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775624.635247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775624.635248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775624.635252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775624.635253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775624.635364:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775624.635369:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775624.635373:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775625.659238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775625.659245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775625.659251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775625.659252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775625.659255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775625.659256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775626.363366:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307181318, at 4307181376 00000100:00000010:0.0:1775775626.363386:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00010000:00000001:3.0:1775775626.683238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775626.683247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775626.683254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775626.683255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775626.683259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775626.683260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775626.683287:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775626.683290:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775626.683294:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775627.579301:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775627.579314:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775627.579318:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775627.579325:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775627.579344:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775627.579353:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12510 00000400:00000200:3.0:1775775627.579365:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775627.579373:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775627.579377:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775627.579380:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775627.579383:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775627.579385:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775627.579388:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775627.579391:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775627.579392:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775627.579395:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775627.579398:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775627.579400:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775627.579402:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775627.579405:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775627.579407:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775627.579408:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775627.579410:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775627.579412:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775627.579415:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775627.579416:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775627.579419:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775627.579422:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775627.579424:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775627.579426:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775627.579443:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775627.579456:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73b80. 00000020:00000040:3.0:1775775627.579461:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775627.579467:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775627.579469:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775627.579472:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775627.579475:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775627.579481:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775627.579492:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775775627.579499:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775627.579511:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73b80 x1862022710353536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775627.579522:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775627.579528:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775627.579543:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c52008f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73b80 x1862022710353536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775627.579569:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775627.579572:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775627.579576:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775627.579577:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775627.579579:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775627.579582:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775627.579584:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12510 00000400:00000200:3.0:1775775627.579588:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775627.579591:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775627.579593:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775627.579596:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775627.579597:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775627.579599:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775627.579601:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775627.579602:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775627.579604:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775627.579605:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775627.579608:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775627.579610:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775627.579612:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775627.579614:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775627.579616:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775627.579617:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775627.579619:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775627.579620:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775627.579622:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775627.579624:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775627.579626:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775627.579632:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775627.579633:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775627.579635:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775627.579640:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775627.579658:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775627.579660:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775627.579682:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775775627.579684:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775627.579686:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775627.579686:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775627.579687:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775627.579689:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775627.579690:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775627.579693:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000100:00000001:3.0:1775775627.579719:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775627.579723:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710353664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775627.579726:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775775627.579731:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775627.579733:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775627.579734:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775627.579738:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775775627.579740:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710353664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775775627.579751:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73b80 x1862022710353536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775627.579763:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775627.579768:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775775627.579769:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775627.579772:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775627.579773:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775627.579775:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775627.579776:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1775775627.579777:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1775775627.579778:0:181953:0:(import.c:484:import_select_connection()) Process entered 02000000:00000001:0.0:1775775627.579781:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1775775627.579783:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12510 02000000:00000001:0.0:1775775627.579786:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775775627.579790:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00100000:0.0:1775775627.579793:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710353536:192.168.201.115@tcp:250:kworker.0 00000400:00000200:3.0:1775775627.579795:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775627.579808:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775775627.579809:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1775775627.579813:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 02000000:00000001:0.0:1775775627.579815:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1775775627.579816:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775775627.579818:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775627.579822:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:0.0:1775775627.579823:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775775627.579827:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775627.579831:0:181953:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000010:0.0:1775775627.579832:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6c00. 00000020:00000040:3.0:1775775627.579833:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 02000000:00000001:0.0:1775775627.579838:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775627.579838:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775627.579842:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775627.579846:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:0.0:1775775627.579849:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000001:3.0:1775775627.579852:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000010:0.0:1775775627.579855:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000100:00000040:3.0:1775775627.579859:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000010:0.0:1775775627.579863:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000001:3.0:1775775627.579863:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775627.579866:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775627.579870:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000200:0.0:1775775627.579873:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710353536, portal 25 00000020:00000040:3.0:1775775627.579874:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00000001:0.0:1775775627.579878:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1775775627.579880:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775627.579881:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849019264 : -123823860532352 : ffff8f6202c73b80) 00000100:00000001:3.0:1775775627.579887:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775627.579890:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73b80 x1862022710353536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775775627.579892:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775627.579898:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775627.579899:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1775775627.579904:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775627.579905:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775627.579910:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000001:3.0:1775775627.579910:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000200:0.0:1775775627.579915:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710353536, offset 0 00000008:00100000:3.0:1775775627.579915:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775627.579922:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775627.579926:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775775627.579926:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775627.579934:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000400:00000200:0.0:1775775627.579938:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000040:3.0:1775775627.579938:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775627.579943:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775627.579945:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775627.579947:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:3.0:1775775627.579949:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775627.579952:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:0.0:1775775627.579954:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1775775627.579955:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775627.579960:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 02000000:00000010:3.0:1775775627.579961:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000400:00000200:0.0:1775775627.579966:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1775775627.579968:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775627.579969:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775627.579978:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453314:453314:256:4294967295] 192.168.201.115@tcp LPNI seq info [453239:453239:8:4294967295] 00000100:00100000:3.0:1775775627.579979:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710353792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775775627.579989:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775775627.579995:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775627.579998:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775627.580005:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710353792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775775627.580008:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:3.0:1775775627.580021:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775775627.580022:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775775627.580024:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:0.0:1775775627.580028:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000100:00000040:3.0:1775775627.580029:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775627.580034:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.0:1775775627.580041:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775775627.580041:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:0.0:1775775627.580048:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00000040:3.0:1775775627.580049:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12520) 00000800:00000200:0.1:1775775627.580054:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775627.580058:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775775627.580061:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775627.580063:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775627.580069:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775627.580073:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775627.580078:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775627.580080:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775627.580084:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775627.580087:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73b80 x1862022710353536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580095:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775627.580097:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73b80 x1862022710353536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775652 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580103:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775627.580105:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775627.580120:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775627.580124:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775627.580126:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307186438 00000100:00000010:0.0:1775775627.580131:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775627.580167:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775627.580174:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307186439 00000100:00000010:0.0:1775775627.580187:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3b40. 00000100:00000001:0.0:1775775627.580190:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775627.580193:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775627.580195:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775627.580197:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775627.580201:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775627/real 1775775627] req@ffff8f6202c73b80 x1862022710353536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775627.580208:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775627.580210:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775627.580212:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775627.580214:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775627.580216:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775627.580217:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775627.580220:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73b80 x1862022710353536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775627.580226:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73b80 x1862022710353536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580231:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775627.580232:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775627.580234:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775627.580236:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775627.580238:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73b80 x1862022710353536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580242:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775627.580245:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73b80 x1862022710353536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580251:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775627.580253:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775627.580254:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775627.580255:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775627.580267:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775627.580269:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775627.580271:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775627.580274:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775627.580275:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775627.580278:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73b80 x1862022710353536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775627.580284:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710353536:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775627.580289:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775627.580291:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775627.580293:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73b80 x1862022710353536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580298:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775627.580300:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775627.580302:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775627.580305:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775627.580306:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775627.580307:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775627.580308:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775627.580310:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34c00. 02000000:00000001:0.0:1775775627.580314:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775627.580316:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775627.580317:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73b80. 00000100:00000001:0.0:1775775627.580320:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775627.580321:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775627.580323:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775627.580329:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775627.580330:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775627.580332:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775627.580333:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775627.580334:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775627.580337:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72300 x1862022710353664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580342:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775627.580343:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775627.580345:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775627.580359:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775627.580361:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775627.580365:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710353664:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775627.580368:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775627.580370:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775627.580372:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775627.580374:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775627.580377:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775627.580379:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775627.580383:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775627.580386:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775627.580390:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775627.580393:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710353664, portal 10 00000100:00000001:0.0:1775775627.580395:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775627.580397:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849012992 : -123823860538624 : ffff8f6202c72300) 00000100:00000040:0.0:1775775627.580405:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72300 x1862022710353664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580413:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775627.580416:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775627.580418:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775627.580421:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710353664, offset 0 00000400:00000200:0.0:1775775627.580425:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775627.580430:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775627.580433:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775627.580435:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775627.580437:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775627.580439:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775627.580441:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775627.580443:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453315:453315:256:4294967295] 192.168.201.115@tcp LPNI seq info [453240:453240:8:4294967295] 00000400:00000200:0.0:1775775627.580446:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775627.580450:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775627.580454:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775627.580456:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775627.580459:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775627.580461:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775775627.580464:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775627.580466:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775775627.580469:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775627.580470:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775627.580473:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775627.580475:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775627.580478:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775627.580479:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775627.580481:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775627.580483:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72300 x1862022710353664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580489:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775627.580491:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72300 x1862022710353664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775652 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580496:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775627.580497:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775627.580499:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775627.580503:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775627.580505:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775627.580506:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307182604 00000100:00000001:0.0:1775775627.580509:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307186438 00000100:00000001:0.0:1775775627.580510:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307182604 00000100:00000001:0.0:1775775627.580512:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307186439 00000100:00000001:0.0:1775775627.580513:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775627.580515:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775627.580516:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775627.580518:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775627/real 1775775627] req@ffff8f6202c72300 x1862022710353664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775627.580524:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775627.580525:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775627.580526:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775627.580528:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775627.580529:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775627.580531:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775627.580533:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72300 x1862022710353664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775627.580539:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72300 x1862022710353664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580544:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775627.580544:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775627.580546:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775627.580546:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775627.580549:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72300 x1862022710353664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580553:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775627.580630:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72300 x1862022710353664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580635:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775627.580637:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775627.580638:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775627.580639:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775627.580640:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775627.580656:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775627.580659:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775627.580662:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775627.580665:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775627.580679:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72300 x1862022710353664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775627.580688:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710353664:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775627.580692:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775627.580696:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72a00 x1862022710353792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580710:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775627.580712:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775627.580715:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775627.580717:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775627.580720:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775627.580723:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710353792:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775627.580727:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775627.580729:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775627.580730:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775627.580732:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775627.580736:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775775627.580738:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775627.580741:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775627.580743:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775627.580745:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775627.580748:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710353792, portal 10 00000100:00000001:0.0:1775775627.580749:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775627.580750:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849014784 : -123823860536832 : ffff8f6202c72a00) 00000100:00000040:0.0:1775775627.580755:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72a00 x1862022710353792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580760:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775627.580761:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775627.580763:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775627.580765:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710353792, offset 0 00000400:00000200:0.0:1775775627.580767:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775627.580770:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775627.580772:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775627.580774:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775627.580776:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775627.580778:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775627.580780:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775627.580782:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453316:453316:256:4294967295] 192.168.201.115@tcp LPNI seq info [453241:453241:8:4294967295] 00000400:00000200:0.0:1775775627.580785:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775627.580790:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775627.580793:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775627.580796:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775627.580799:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775627.580801:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775775627.580804:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775627.580806:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775775627.580809:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775627.580811:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775627.580813:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775627.580815:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775627.580817:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775627.580818:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775627.580820:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775627.580822:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72a00 x1862022710353792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580828:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775627.580830:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72a00 x1862022710353792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775652 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580835:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775627.580836:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775627.580837:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775627.580840:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775627.580842:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775627.580843:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307182604 00000100:00000001:0.0:1775775627.580845:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307186438 00000100:00000001:0.0:1775775627.580847:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307182604 00000100:00000001:0.0:1775775627.580849:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307186439 00000100:00000001:0.0:1775775627.580850:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775627.580852:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775627.580854:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775627.580856:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775627/real 1775775627] req@ffff8f6202c72a00 x1862022710353792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775627.580862:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775627.580862:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775627.580863:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775627.580865:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775627.580866:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775627.580867:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775627.580869:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72a00 x1862022710353792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775627.580875:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72a00 x1862022710353792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580879:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775627.580880:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775627.580881:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775627.580882:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775627.580884:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72a00 x1862022710353792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580888:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775627.580891:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72a00 x1862022710353792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580895:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775627.580896:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775627.580897:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775627.580898:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775627.580899:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775627.580900:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775627.580902:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775627.580905:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775627.580906:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775627.580909:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72a00 x1862022710353792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775627.580914:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710353792:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775627.580916:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775627.580918:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775627.580919:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72300 x1862022710353664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580923:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775627.580924:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775627.580926:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775627.580927:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775627.580928:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775627.580929:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775627.580931:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775627.580932:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36400. 02000000:00000001:0.0:1775775627.580934:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775627.580935:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775627.580937:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72300. 00000100:00000001:0.0:1775775627.580939:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775627.580940:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775627.580941:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775627.580943:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72a00 x1862022710353792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775652 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775627.580946:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775627.580947:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775627.580948:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775775627.580949:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775627.580950:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775627.580951:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775627.580952:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775627.580954:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36000. 02000000:00000001:0.0:1775775627.580955:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775627.580956:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775627.580957:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72a00. 00000100:00000001:0.0:1775775627.580958:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775627.580959:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775627.580960:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775627.580963:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775627.580964:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775627.580965:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775627.580966:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775627.707257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775627.707265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775627.707273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775627.707277:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775627.707279:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775627.707283:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775627.707286:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775627.707288:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775627.707290:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775627.707293:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775627.707297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775627.707298:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775627.707299:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775627.707301:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775627.707302:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775627.707304:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775627.707305:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775627.707308:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775627.707355:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775627.707360:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775627.707363:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775628.731269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775628.731275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775628.731282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775628.731284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775628.731298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775628.731300:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775629.755239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775629.755247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775629.755254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775629.755255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775629.755260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775629.755262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775629.755346:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775629.755350:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775629.755354:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775630.779261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775630.779267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775630.779272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775630.779273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775630.779291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775630.779292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775631.483237:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307186438, at 4307186496 00000100:00000010:0.0:1775775631.483252:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00010000:00000001:3.0:1775775631.803251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775631.803257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775631.803265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775631.803267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775631.803273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775631.803275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775631.803297:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775631.803301:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775631.803305:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775775632.699544:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775632.699549:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775632.699553:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775632.699599:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491aa00 x1862022710353920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.699609:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775632.699611:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775632.699617:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775632.699621:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775632.699626:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775632.699633:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710353920:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775632.699640:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775632.699643:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775632.699646:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775632.699651:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775632.699657:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775775632.699661:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775632.699669:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775632.699675:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775632.699682:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775632.699690:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710353920, portal 25 00000100:00000001:0.0:1775775632.699694:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775632.699697:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684369408 : -123823025182208 : ffff8f623491aa00) 00000100:00000040:0.0:1775775632.699707:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491aa00 x1862022710353920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775657 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.699717:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775632.699723:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775632.699726:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775632.699729:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710353920, offset 0 00000400:00000200:0.0:1775775632.699734:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775632.699743:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775632.699751:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775632.699758:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775632.699761:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775632.699764:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775632.699766:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775632.699769:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453317:453317:256:4294967295] 192.168.201.115@tcp LPNI seq info [453242:453242:8:4294967295] 00000400:00000200:0.0:1775775632.699772:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775632.699778:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775632.699786:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775632.699792:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775632.699804:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775632.699809:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775775632.699819:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775632.699825:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775775632.699831:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775632.699835:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775632.699842:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775632.699853:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775632.699859:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775632.699862:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775632.699868:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775632.699875:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491aa00 x1862022710353920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775657 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.699891:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775632.699895:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491aa00 x1862022710353920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775657 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.699911:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775632.699914:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775632.699921:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775632.699927:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775632.699930:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307191558 00000100:00000010:0.0:1775775632.699936:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775632.699945:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775632.699952:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307191558 00000100:00000010:0.0:1775775632.699956:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c00. 00000100:00000001:0.0:1775775632.699959:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775632.699963:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775632.699969:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775632.699972:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775632.699978:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775632/real 1775775632] req@ffff8f623491aa00 x1862022710353920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775632.699991:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775632.699993:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775632.699995:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775632.699999:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775632.700001:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775632.700004:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775632.700007:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491aa00 x1862022710353920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775632.700013:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491aa00 x1862022710353920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.700018:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775632.700020:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775632.700021:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775632.700022:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775632.700025:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491aa00 x1862022710353920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.700029:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775632.700032:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491aa00 x1862022710353920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.700037:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775632.700038:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775632.700039:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775632.700040:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775632.700042:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775632.700043:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775632.700045:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775632.700048:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775632.700049:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775632.700052:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491aa00 x1862022710353920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775632.700057:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710353920:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775632.700060:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775632.700061:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775632.700063:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491aa00 x1862022710353920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.700067:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775632.700069:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775632.700071:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775775632.700073:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775632.700074:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775632.700076:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775632.700077:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775632.700079:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:0.0:1775775632.700082:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775632.700083:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775632.700084:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491aa00. 00000100:00000001:0.0:1775775632.700086:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775632.700087:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775632.700089:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775632.700093:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775632.700094:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775632.700096:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775632.700097:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775632.700098:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775632.700100:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918e00 x1862022710354048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.700106:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775632.700107:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775632.700108:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775632.700109:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775632.700111:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775632.700112:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710354048:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775632.700114:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775632.700115:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775632.700116:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775632.700117:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775632.700119:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775775632.700120:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775632.700122:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775632.700124:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775632.700126:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775632.700129:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710354048, portal 10 00000100:00000001:0.0:1775775632.700130:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775632.700131:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684362240 : -123823025189376 : ffff8f6234918e00) 00000100:00000040:0.0:1775775632.700136:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918e00 x1862022710354048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775657 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.700144:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775632.700147:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775632.700178:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775632.700182:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710354048, offset 0 00000400:00000200:0.0:1775775632.700187:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775632.700192:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775632.700196:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775632.700199:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775632.700201:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775632.700219:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775632.700221:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775632.700224:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453318:453318:256:4294967295] 192.168.201.115@tcp LPNI seq info [453243:453243:8:4294967295] 00000400:00000200:0.0:1775775632.700233:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775632.700239:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775632.700242:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775632.700246:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775632.700249:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775632.700251:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775775632.700255:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775632.700257:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775775632.700260:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775632.700261:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775632.700264:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775632.700267:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775632.700269:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775632.700271:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775632.700273:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775632.700276:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918e00 x1862022710354048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775657 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.700287:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775632.700289:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918e00 x1862022710354048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775657 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.700299:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775632.700300:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775632.700303:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775632.700306:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775632.700308:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775632.700310:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307187724 00000100:00000001:0.0:1775775632.700312:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307191558 00000100:00000001:0.0:1775775632.700314:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307187724 00000100:00000001:0.0:1775775632.700316:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307191558 00000100:00000001:0.0:1775775632.700317:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775632.700319:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775632.700320:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775632.700323:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775632/real 1775775632] req@ffff8f6234918e00 x1862022710354048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775632.700329:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775632.700330:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775632.700331:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775632.700333:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775632.700334:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775632.700335:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775632.700337:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918e00 x1862022710354048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775632.700344:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918e00 x1862022710354048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.700348:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775632.700349:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775632.700350:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775632.700351:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775632.700353:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918e00 x1862022710354048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.700359:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775632.700362:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918e00 x1862022710354048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.700367:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775632.700368:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775632.700370:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775632.700370:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775632.700372:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775632.700375:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775632.700377:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775632.700379:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775632.700381:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775632.700383:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918e00 x1862022710354048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775632.700389:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710354048:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775632.700391:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775632.700394:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918380 x1862022710354176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.700399:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775632.700400:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775632.700401:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775632.700403:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775632.700404:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775632.700406:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710354176:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775632.700408:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775632.700410:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775632.700411:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775632.700413:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775632.700415:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775632.700416:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775632.700419:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775632.700420:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775632.700423:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775632.700425:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710354176, portal 10 00000100:00000001:0.0:1775775632.700426:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775632.700427:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684359552 : -123823025192064 : ffff8f6234918380) 00000100:00000040:0.0:1775775632.700432:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918380 x1862022710354176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775657 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.700436:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775632.700438:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775632.700440:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775632.700441:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710354176, offset 0 00000400:00000200:0.0:1775775632.700460:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775632.700463:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775632.700465:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775632.700467:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775632.700468:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775632.700470:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775632.700472:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775632.700474:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453319:453319:256:4294967295] 192.168.201.115@tcp LPNI seq info [453244:453244:8:4294967295] 00000400:00000200:0.0:1775775632.700477:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775632.700481:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775632.700484:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775632.700486:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775632.700488:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775632.700490:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775775632.700493:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775632.700495:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775775632.700497:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775632.700499:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775632.700501:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775632.700503:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775632.700505:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775632.700506:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775632.700508:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775632.700525:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918380 x1862022710354176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775657 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.700531:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775632.700533:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918380 x1862022710354176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775657 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.700538:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775632.700539:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775632.700541:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775632.700543:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775632.700545:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775632.700546:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307187724 00000100:00000001:0.0:1775775632.700548:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307191558 00000100:00000001:0.0:1775775632.700550:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307187724 00000100:00000001:0.0:1775775632.700551:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307191558 00000100:00000001:0.0:1775775632.700569:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775632.700571:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775632.700573:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775632.700576:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775632/real 1775775632] req@ffff8f6234918380 x1862022710354176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775632.700582:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775632.700583:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775632.700585:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775632.700586:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775632.700588:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775632.700590:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775632.700592:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918380 x1862022710354176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775632.700598:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918380 x1862022710354176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.700602:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775632.700603:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775632.700604:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775632.700605:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775632.700608:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918380 x1862022710354176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.700612:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775632.700615:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918380 x1862022710354176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.700620:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775632.700621:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775632.700623:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775632.700623:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775632.700625:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775632.700626:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775632.700628:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775632.700630:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775632.700631:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775632.700634:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918380 x1862022710354176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775632.700639:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710354176:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775632.700641:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775632.700643:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775632.700645:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918e00 x1862022710354048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.700650:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775632.700652:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775632.700653:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775775632.700655:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775632.700656:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775632.700657:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775632.700659:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775632.700660:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4c000. 02000000:00000001:0.0:1775775632.700662:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775632.700663:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775632.700664:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918e00. 00000100:00000001:0.0:1775775632.700666:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775632.700667:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775632.700668:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775632.700670:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918380 x1862022710354176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775657 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775632.700676:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775632.700677:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775632.700678:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775632.700679:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775632.700680:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775632.700681:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775632.700682:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775632.700683:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:0.0:1775775632.700684:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775632.700685:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775632.700686:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918380. 00000100:00000001:0.0:1775775632.700687:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775632.700688:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775632.700689:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775632.700692:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775632.700693:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775632.700694:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775632.700696:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775632.827263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775632.827269:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775632.827271:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775632.827274:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775632.827278:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775632.827280:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775632.827281:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775632.827283:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775632.827285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775632.827286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775632.827290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775632.827291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775632.827374:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775632.827377:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775632.827380:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775633.851224:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775633.851229:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775633.851234:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775633.851235:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775633.851238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775633.851239:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775634.875258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775634.875264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775634.875270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775634.875272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775634.875276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775634.875278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775634.875350:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775634.875354:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775634.875357:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775635.899262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775635.899268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775635.899274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775635.899275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775635.899280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775635.899281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775636.603225:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307191558, at 4307191616 00000100:00000010:0.0:1775775636.603245:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00010000:00000001:3.0:1775775636.923240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775636.923246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775636.923249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775636.923251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775636.923253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775636.923254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775636.923273:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775636.923275:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775636.923277:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775637.819311:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775637.819321:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775637.819325:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775637.819335:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775637.819340:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775637.819349:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12520 00000400:00000200:2.0:1775775637.819360:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775637.819368:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775637.819371:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775637.819375:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775637.819378:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775637.819383:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775637.819385:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775637.819390:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775637.819391:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775637.819394:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775637.819397:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775637.819399:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775637.819401:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775637.819404:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775637.819405:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775637.819407:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775637.819408:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775637.819411:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775637.819414:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775637.819416:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775637.819418:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775637.819421:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775637.819423:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775637.819425:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775637.819427:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775637.819435:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775775637.819438:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775637.819442:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775637.819443:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775637.819445:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775637.819448:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775637.819452:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775637.819460:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775775637.819480:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775637.819489:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710354304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775637.819499:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775637.819519:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775637.819523:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710354304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775637.819539:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775637.819541:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775637.819544:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775637.819545:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775637.819546:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775637.819548:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775637.819549:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12520 00000400:00000200:2.0:1775775637.819552:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775637.819554:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775637.819555:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775637.819557:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775637.819559:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775637.819560:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775637.819561:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775637.819563:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775637.819564:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775637.819565:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775637.819567:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775637.819568:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775637.819570:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775637.819572:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775637.819574:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775637.819576:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775637.819578:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775637.819579:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775637.819581:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775637.819584:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775637.819585:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775775637.819589:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1775775637.819590:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775637.819592:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775637.819595:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775637.819596:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000008:00100000:2.0:1775775637.819598:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1775775637.819600:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000008:00000001:2.0:1775775637.819603:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775637.819606:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775637.819610:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344bb80 x1862022710354304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775775637.819610:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775775637.819613:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775637.819617:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775637.819619:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775637.819623:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775637.819626:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775637.819628:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775637.819633:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000010:2.0:1775775637.819633:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:0.0:1775775637.819635:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775637.819637:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775637.819641:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:2.0:1775775637.819643:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710354432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1775775637.819646:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775637.819652:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775637.819659:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1775775637.819661:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710354304:192.168.201.115@tcp:250:kworker.0 00000100:00000001:2.0:1775775637.819662:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775637.819668:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775637.819671:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:2.0:1775775637.819672:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710354432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775775637.819674:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775637.819678:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775637.819685:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 00000100:00000001:2.0:1775775637.819687:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775637.819688:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775637.819693:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775775637.819698:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00080000:2.0:1775775637.819701:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000010:0.0:1775775637.819703:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000100:00000001:2.0:1775775637.819704:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775637.819707:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:0.0:1775775637.819710:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000001:2.0:1775775637.819711:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775637.819716:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12520 00000100:00000200:0.0:1775775637.819717:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710354304, portal 25 00000100:00000001:0.0:1775775637.819720:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000400:00000200:2.0:1775775637.819721:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775637.819722:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662550400 : -123823047001216 : ffff8f623344bb80) 00000400:00000200:2.0:1775775637.819727:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000040:0.0:1775775637.819731:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344bb80 x1862022710354304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775662 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775775637.819731:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775637.819738:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775637.819743:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775775637.819746:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775775637.819747:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775637.819750:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:0.0:1775775637.819754:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000020:00000001:2.0:1775775637.819754:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775637.819757:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000400:00000010:0.0:1775775637.819758:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000020:00000001:2.0:1775775637.819759:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775637.819762:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000100:00000200:0.0:1775775637.819763:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710354304, offset 0 00000020:00000001:2.0:1775775637.819764:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775637.819767:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:0.0:1775775637.819771:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000040:2.0:1775775637.819773:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775637.819776:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775637.819777:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775637.819779:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:0.0:1775775637.819781:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000040:2.0:1775775637.819783:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775637.819786:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775637.819788:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775637.819790:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1775775637.819791:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 02000000:00000001:2.0:1775775637.819793:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775637.819796:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775637.819798:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775637.819799:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1775775637.819800:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000008:00000001:2.0:1775775637.819803:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775637.819805:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775637.819808:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000010:2.0:1775775637.819810:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775775637.819812:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:0.0:1775775637.819815:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 02000000:00000001:2.0:1775775637.819816:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775637.819818:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775637.819820:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1775775637.819823:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 02000000:00000001:2.0:1775775637.819823:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775637.819826:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775637.819831:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453320:453320:256:4294967295] 192.168.201.115@tcp LPNI seq info [453245:453245:8:4294967295] 00000400:00000200:0.0:1775775637.819837:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000010:2.0:1775775637.819838:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775775637.819842:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775637.819852:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710354560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775775637.819853:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:2.0:1775775637.819864:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:0.0:1775775637.819866:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:2.0:1775775637.819867:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775637.819873:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710354560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:0.0:1775775637.819875:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000100:00000001:2.0:1775775637.819881:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775637.819902:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.0:1775775637.819907:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775775637.819917:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:0.0:1775775637.819919:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000100:00000040:2.0:1775775637.819921:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775637.819929:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775637.819936:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12530) 00000800:00000200:0.1:1775775637.819936:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775637.819943:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775775637.819950:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775637.819954:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775637.819961:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775637.819979:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775637.819985:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775637.819989:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775637.819994:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775637.820001:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344bb80 x1862022710354304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775662 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820014:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775637.820019:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344bb80 x1862022710354304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775662 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820029:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775637.820032:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775637.820040:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775637.820045:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775637.820049:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307196678 00000100:00000010:0.0:1775775637.820056:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775637.820064:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775637.820082:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307196678 00000100:00000010:0.0:1775775637.820085:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775775637.820088:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775637.820091:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775637.820094:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775637.820096:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775637.820101:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775637/real 1775775637] req@ffff8f623344bb80 x1862022710354304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775637.820120:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775637.820122:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775637.820124:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775637.820127:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775637.820128:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775637.820131:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775637.820164:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344bb80 x1862022710354304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775637.820189:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344bb80 x1862022710354304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820199:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775637.820201:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775637.820204:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775637.820207:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775637.820212:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344bb80 x1862022710354304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820222:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775637.820227:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344bb80 x1862022710354304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820238:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775637.820242:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775637.820244:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775637.820246:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775637.820249:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775637.820253:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775637.820257:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775637.820262:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775637.820265:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775637.820271:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344bb80 x1862022710354304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775637.820284:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710354304:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775637.820289:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775637.820292:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775637.820296:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344bb80 x1862022710354304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820306:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775637.820309:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775637.820312:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775775637.820318:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775637.820320:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775637.820322:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775637.820325:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775637.820328:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4d800. 02000000:00000001:0.0:1775775637.820332:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775637.820335:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775637.820337:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344bb80. 00000100:00000001:0.0:1775775637.820342:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775637.820344:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775637.820347:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775637.820356:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775637.820358:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775637.820361:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775637.820363:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775637.820365:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775637.820371:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233448700 x1862022710354432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820381:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775637.820383:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775637.820386:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775637.820389:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775637.820392:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775637.820397:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710354432:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775637.820402:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775637.820405:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775637.820407:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775637.820409:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775637.820413:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775775637.820417:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775637.820422:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775637.820424:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775637.820428:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775637.820431:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710354432, portal 10 00000100:00000001:0.0:1775775637.820434:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775637.820435:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662536960 : -123823047014656 : ffff8f6233448700) 00000100:00000040:0.0:1775775637.820441:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233448700 x1862022710354432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775662 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820448:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775637.820450:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775637.820452:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775637.820454:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710354432, offset 0 00000400:00000200:0.0:1775775637.820458:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775637.820480:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775637.820504:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775637.820518:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775637.820520:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775637.820523:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775637.820525:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775637.820529:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453321:453321:256:4294967295] 192.168.201.115@tcp LPNI seq info [453246:453246:8:4294967295] 00000400:00000200:0.0:1775775637.820533:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775637.820541:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775637.820546:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775637.820550:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775637.820555:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775637.820558:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775775637.820562:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775637.820565:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775775637.820569:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775637.820570:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775637.820573:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775637.820575:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775637.820578:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775637.820579:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775637.820581:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775637.820584:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233448700 x1862022710354432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775662 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820589:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775637.820591:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448700 x1862022710354432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775662 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820596:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775637.820597:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775637.820599:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775637.820602:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775637.820604:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775637.820606:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307192844 00000100:00000001:0.0:1775775637.820608:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307196678 00000100:00000001:0.0:1775775637.820610:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307192844 00000100:00000001:0.0:1775775637.820612:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307196678 00000100:00000001:0.0:1775775637.820613:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775637.820614:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775637.820616:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775637.820618:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775637/real 1775775637] req@ffff8f6233448700 x1862022710354432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775637.820623:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775637.820623:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775637.820625:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775637.820626:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775637.820627:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775637.820629:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775637.820630:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233448700 x1862022710354432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775637.820636:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233448700 x1862022710354432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820639:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775637.820640:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775637.820641:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775637.820642:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775637.820644:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233448700 x1862022710354432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820648:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775637.820651:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233448700 x1862022710354432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820656:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775637.820657:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775637.820658:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775637.820659:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775637.820661:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775637.820664:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775637.820666:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775637.820668:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775637.820670:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775637.820672:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233448700 x1862022710354432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775637.820677:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710354432:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775637.820679:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775637.820681:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449c00 x1862022710354560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820685:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775637.820686:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775637.820687:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775637.820688:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775637.820689:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775637.820691:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710354560:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775637.820693:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775637.820695:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775637.820696:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775637.820697:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775637.820699:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775637.820700:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775637.820702:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775637.820704:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775637.820706:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775637.820708:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710354560, portal 10 00000100:00000001:0.0:1775775637.820709:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775637.820711:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662542336 : -123823047009280 : ffff8f6233449c00) 00000100:00000040:0.0:1775775637.820714:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449c00 x1862022710354560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775662 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820718:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775637.820720:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775637.820721:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775637.820723:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710354560, offset 0 00000400:00000200:0.0:1775775637.820725:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775637.820728:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775637.820730:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775637.820732:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775637.820733:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775637.820735:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775637.820737:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775637.820739:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453322:453322:256:4294967295] 192.168.201.115@tcp LPNI seq info [453247:453247:8:4294967295] 00000400:00000200:0.0:1775775637.820742:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775637.820746:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775637.820749:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775637.820751:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775637.820754:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775637.820756:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775775637.820758:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775637.820760:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775775637.820762:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775637.820763:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775637.820765:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775637.820767:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775637.820769:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775637.820770:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775637.820772:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775637.820774:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449c00 x1862022710354560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775662 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820782:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775637.820784:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449c00 x1862022710354560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775662 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820788:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775637.820789:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775637.820791:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775637.820792:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775637.820794:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775637.820795:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307192844 00000100:00000001:0.0:1775775637.820797:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307196678 00000100:00000001:0.0:1775775637.820798:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307192844 00000100:00000001:0.0:1775775637.820800:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307196678 00000100:00000001:0.0:1775775637.820801:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775637.820802:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775637.820804:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775637.820806:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775637/real 1775775637] req@ffff8f6233449c00 x1862022710354560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775637.820810:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775637.820811:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775637.820812:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775637.820813:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775637.820814:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775637.820815:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775637.820817:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449c00 x1862022710354560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775637.820822:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449c00 x1862022710354560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820826:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775637.820828:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775637.820829:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775637.820830:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775637.820832:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449c00 x1862022710354560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820836:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775637.820838:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449c00 x1862022710354560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820842:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775637.820843:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775637.820844:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775637.820845:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775637.820846:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775637.820847:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775637.820849:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775637.820850:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775637.820852:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775637.820854:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449c00 x1862022710354560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775637.820858:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710354560:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775637.820860:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775637.820861:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775637.820863:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233448700 x1862022710354432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820868:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775637.820869:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775637.820870:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775775637.820871:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775637.820872:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775637.820874:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775637.820875:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775637.820876:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4f000. 02000000:00000001:0.0:1775775637.820878:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775637.820879:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775637.820880:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233448700. 00000100:00000001:0.0:1775775637.820881:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775637.820882:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775637.820883:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775637.820885:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449c00 x1862022710354560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775662 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775637.820889:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775637.820890:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775637.820891:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775775637.820892:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775637.820893:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775637.820894:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775637.820895:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775637.820896:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4cc00. 02000000:00000001:0.0:1775775637.820897:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775637.820898:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775637.820898:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449c00. 00000100:00000001:0.0:1775775637.820900:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775637.820900:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775637.820901:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775637.820904:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775637.820905:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775637.820906:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775637.820907:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775637.947240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775637.947247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775637.947253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775637.947257:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775637.947260:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775637.947263:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775637.947266:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775637.947268:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775637.947270:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775637.947272:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775637.947276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775637.947277:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775637.947278:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775637.947280:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775637.947281:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775637.947283:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775637.947284:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775637.947286:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775637.947361:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775637.947365:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775637.947368:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775638.971213:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775638.971218:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775638.971222:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775638.971223:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775638.971226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775638.971227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775639.995293:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775639.995299:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775639.995305:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775639.995307:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775639.995311:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775639.995312:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775639.995458:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775639.995463:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775639.995484:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775641.019249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775641.019253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775641.019258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775641.019259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775641.019262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775641.019263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775641.723299:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307196678, at 4307196736 00000100:00000010:0.0:1775775641.723315:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00010000:00000001:3.0:1775775642.043264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775642.043272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775642.043277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775642.043278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775642.043281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775642.043282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775642.043324:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775642.043327:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775642.043329:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775642.939223:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775642.939230:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775642.939233:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775642.939236:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775642.939239:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775642.939245:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12525 00000400:00000200:0.0:1775775642.939253:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775642.939257:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775642.939260:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775642.939262:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775642.939264:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775642.939266:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775642.939268:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775642.939271:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775642.939273:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775642.939275:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775642.939277:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775642.939278:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775642.939280:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775642.939282:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775642.939284:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775642.939285:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775642.939286:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775642.939288:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775642.939290:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775642.939292:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775642.939294:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775642.939296:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775642.939298:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775642.939300:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775642.939301:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775642.939308:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063100. 00000020:00000040:0.0:1775775642.939310:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775642.939314:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775642.939316:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775642.939318:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775642.939320:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775642.939323:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775642.939330:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6c00. 00000100:00000001:0.0:1775775642.939335:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775642.939342:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063100 x1862022710354688/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775642.939349:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775642.939353:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775642.939357:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033f6db7f] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063100 x1862022710354688/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775642.939387:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775642.939390:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775642.939392:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775642.939398:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775642.939399:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775642.939402:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775642.939404:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12525 00000400:00000200:0.0:1775775642.939411:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775642.939417:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775642.939421:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775642.939428:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775642.939433:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775642.939437:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775642.939441:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775642.939444:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775642.939446:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775642.939449:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775642.939452:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775642.939458:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775642.939466:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775642.939471:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775642.939474:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775642.939477:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775642.939478:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775642.939480:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775642.939482:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775642.939484:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775642.939488:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775642.939493:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775642.939496:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775642.939500:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775642.939505:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775642.939511:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775642.939514:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775642.939517:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060e00. 00000020:00000040:0.0:1775775642.939520:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775642.939523:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775642.939525:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775642.939527:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775642.939528:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775642.939530:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775642.939541:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5800. 00000100:00000001:0.0:1775775642.939545:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775642.939553:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060e00 x1862022710354816/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775642.939569:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775642.939573:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775642.939580:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004a3bddaa] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060e00 x1862022710354816/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775642.939613:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775642.939617:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775642.939622:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775642.939625:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775642.939628:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775642.939632:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775642.939638:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12525 00000400:00000200:0.0:1775775642.939675:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775642.939684:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775642.939689:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775642.939695:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775642.939701:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775642.939705:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775642.939708:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775642.939712:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000800:00000200:1.1:1775775642.939712:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000040:0.0:1775775642.939714:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775642.939718:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:1.1:1775775642.939718:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000020:00000040:0.0:1775775642.939720:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775642.939723:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775642.939727:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775642.939737:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775642.939742:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775642.939747:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775642.939751:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775642.939757:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775642.939762:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775642.939768:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775642.939774:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775642.939779:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775642.939785:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775642.939788:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775642.939793:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775642.939800:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775642.939806:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775642.939814:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061c00. 00000020:00000040:0.0:1775775642.939820:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775642.939827:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775642.939830:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775642.939833:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775642.939837:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775642.939841:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775642.939849:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6400. 00000100:00000001:0.0:1775775642.939856:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775642.939867:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061c00 x1862022710354944/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775642.939885:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775642.939890:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775642.939900:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076a44879] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061c00 x1862022710354944/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775642.939915:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775642.939920:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775642.939926:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775642.939929:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775642.939934:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775642.939940:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12535) 00000800:00000200:1.1:1775775642.940170:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775642.940172:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775775642.940445:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775642.940447:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00010000:00000001:3.0:1775775643.067287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775643.067298:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775643.067301:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775643.067306:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775643.067313:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775643.067316:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775643.067319:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775643.067322:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775643.067327:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775643.067329:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775643.067334:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775643.067335:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775643.067444:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775643.067449:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775643.067453:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.0:1775775643.771212:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775775643.771218:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775775643.771222:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1775775643.771416:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775775643.771434:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775775643.771440:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775775643.771442:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1775775644.091221:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775644.091227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775644.091233:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775644.091235:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775644.091238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775644.091240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775645.115269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775645.115276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775645.115281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775645.115282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775645.115286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775645.115287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775645.115379:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775645.115385:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775645.115389:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775646.139226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775646.139231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775646.139235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775646.139236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775646.139239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775646.139240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775646.843289:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307201798, at 4307201856 00000100:00000010:0.0:1775775646.843304:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2a80. 00010000:00000001:3.0:1775775647.163242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775647.163250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775647.163256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775647.163258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775647.163262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775647.163263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775647.163290:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775647.163293:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775647.163296:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775648.059277:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775648.059305:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775648.059310:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775648.059315:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775648.059322:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775648.059331:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12530 00000400:00000200:3.0:1775775648.059343:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775648.059352:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775648.059357:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775648.059360:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775648.059363:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775648.059365:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775648.059367:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775648.059371:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775648.059373:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775648.059376:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775648.059378:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775648.059380:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775648.059382:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775648.059385:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775648.059393:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775648.059395:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775648.059396:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775648.059399:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775648.059402:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775648.059403:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775648.059406:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775648.059409:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775648.059412:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775648.059414:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775648.059415:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775648.059427:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775775648.059430:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775648.059434:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775648.059436:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775648.059439:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775648.059442:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775648.059447:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775648.059456:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000100:00000001:3.0:1775775648.059463:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775648.059474:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710355072/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775648.059493:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775648.059499:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775648.059506:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710355072/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775648.059536:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775648.059538:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775648.059541:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775648.059543:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775648.059545:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775648.059548:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775648.059551:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12530 00000400:00000200:3.0:1775775648.059555:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775648.059559:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775648.059561:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775648.059565:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775648.059568:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775648.059578:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775648.059581:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775648.059585:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775648.059590:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775648.059592:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775648.059597:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775648.059605:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775648.059614:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775648.059626:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775648.059631:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775648.059637:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775648.059640:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775648.059643:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775648.059655:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775648.059668:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775648.059671:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775648.059678:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775648.059684:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775648.059687:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775648.059692:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775648.059701:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775648.059704:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775648.059713:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000020:00000040:3.0:1775775648.059717:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775648.059720:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775648.059721:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775648.059723:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775648.059725:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775648.059728:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775648.059732:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000100:00000001:3.0:1775775648.059739:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775648.059758:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710355200/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775648.059802:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775648.059813:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775648.059823:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710355200/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775648.059850:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775648.059856:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775648.059862:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775648.059868:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775648.059873:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775648.059884:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775648.059896:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12530 00000400:00000200:3.0:1775775648.059915:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775648.059932:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775648.059939:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775648.059946:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775648.059949:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775648.059954:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775648.059966:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000200:1.1:1775775648.059971:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:3.0:1775775648.059977:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775648.059987:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775648.059990:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775648.059992:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775648.059995:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775648.060004:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775648.060019:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775648.060025:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775648.060030:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775648.060033:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775648.060048:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775648.060059:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775648.060069:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775648.060075:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775648.060081:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775648.060086:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775648.060097:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775648.060101:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775648.060108:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775648.060112:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775648.060118:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775775648.060127:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775648.060176:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775648.060178:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775648.060183:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775648.060186:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775648.060194:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775648.060206:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775775648.060221:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775648.060232:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710355328/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775648.060260:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775648.060264:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775648.060273:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710355328/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775648.060313:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775648.060327:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775648.060340:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775648.060346:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775648.060359:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775648.060372:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12540) 00000800:00000200:1.1:1775775648.060422:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775648.060582:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00010000:00000001:3.0:1775775648.187252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775648.187259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775648.187265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775648.187268:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775648.187270:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775648.187273:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775648.187277:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775648.187279:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775648.187282:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775648.187297:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775648.187303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775648.187304:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775648.187306:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775648.187308:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775648.187310:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775648.187312:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775648.187314:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775648.187316:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775648.187391:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775648.187399:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775648.187403:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775649.211237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775649.211244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775649.211250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775649.211251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775649.211253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775649.211254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775650.235276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775650.235283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775650.235289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775650.235290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775650.235293:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775650.235294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775650.235389:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775650.235396:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775650.235402:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775775651.131571:0:2399:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:3.0:1775775651.131636:0:2399:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1775775651.131650:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775651.131655:0:2399:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:3.0:1775775651.131663:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775651.131669:0:2399:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:3.0:1775775651.131675:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775775651.131681:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:3.0:1775775651.131688:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775775651.131710:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775775651.131719:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3aa0 00000400:00000010:3.0:1775775651.131723:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3aa0. 00000100:00000001:3.0:1775775651.131732:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775775651.131742:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c71180 x1862022710355072/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775673 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775651.131771:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775775651.131776:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71180 x1862022710355072/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775673 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775651.131785:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775651.131787:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775775651.131807:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775775651.131810:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230700. 00000400:00000200:3.0:1775775651.131814:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775775651.131824:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775775651.131830:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3770 00000400:00000010:3.0:1775775651.131833:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3770. 00000100:00000001:3.0:1775775651.131846:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775775651.131866:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c70a80 x1862022710355200/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775673 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775651.131904:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775775651.131912:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70a80 x1862022710355200/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775673 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775651.131929:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775651.131933:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775775651.131945:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775775651.131953:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230a00. 00000400:00000200:3.0:1775775651.131959:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775775651.131976:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775775651.131983:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3b28 00000400:00000010:3.0:1775775651.131987:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3b28. 00000100:00000001:3.0:1775775651.132001:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775775651.132012:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c70700 x1862022710355328/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775673 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775651.132068:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775775651.132078:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70700 x1862022710355328/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775673 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775651.132105:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775651.132110:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:3.0:1775775651.259224:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775651.259255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775651.259264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775651.259265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775651.259268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775651.259269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775651.963283:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307206918, at 4307206976 00000100:00000010:0.0:1775775651.963305:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2720. 00010000:00000001:3.0:1775775652.283255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775652.283260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775652.283265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775652.283266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775652.283269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775652.283270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775652.283296:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775652.283298:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775652.283300:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775653.179244:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775653.179261:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775653.179270:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775653.179275:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775653.179281:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775653.179289:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12535 00000400:00000200:3.0:1775775653.179300:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775653.179307:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775653.179311:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775653.179315:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775653.179318:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775653.179321:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775653.179323:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775653.179327:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775653.179329:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775653.179331:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775653.179334:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775653.179337:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775653.179340:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775653.179343:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775653.179345:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775653.179347:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775653.179349:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775653.179352:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775653.179355:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775653.179357:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775653.179361:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775653.179365:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775653.179367:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775653.179370:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775653.179372:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775653.179379:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775775653.179383:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775653.179387:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775653.179389:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775653.179391:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775653.179394:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775653.179398:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775653.179406:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000001:3.0:1775775653.179413:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775653.179423:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710355456/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775653.179434:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775653.179439:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775653.179444:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710355456/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775653.179476:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775653.179480:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775653.179483:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775653.179485:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775653.179487:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775653.179490:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775653.179492:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12535 00000400:00000200:3.0:1775775653.179496:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775653.179500:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775653.179502:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775653.179505:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775653.179507:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775653.179510:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775653.179512:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775653.179514:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775653.179516:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775653.179517:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775653.179519:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775653.179521:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775653.179523:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775653.179526:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775653.179527:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775653.179529:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775653.179531:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775653.179533:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775653.179536:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775653.179538:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775653.179540:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775653.179546:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775653.179548:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775653.179549:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775653.179568:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775653.179572:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775653.179573:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775653.179578:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775775653.179580:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775653.179582:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775653.179584:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775653.179585:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775653.179587:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775653.179588:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775653.179592:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000100:00000001:3.0:1775775653.179595:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775653.179601:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710355584/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775653.179609:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775653.179611:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775653.179615:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710355584/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775653.179623:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775653.179626:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775653.179629:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775653.179631:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775653.179633:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775653.179636:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775653.179639:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12535 00000400:00000200:3.0:1775775653.179643:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775653.179646:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775653.179649:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775653.179651:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775653.179654:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775653.179655:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775653.179658:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775653.179660:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775653.179661:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775653.179662:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775653.179665:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775653.179667:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775653.179668:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775653.179671:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775653.179673:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775653.179675:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775653.179677:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775653.179680:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775653.179682:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775653.179685:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775653.179687:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775653.179691:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775653.179693:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775653.179695:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775653.179698:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775653.179700:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775653.179702:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775653.179705:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775775653.179708:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775653.179711:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775653.179712:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775653.179714:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775653.179715:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775653.179717:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775653.179721:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000001:3.0:1775775653.179724:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775653.179730:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710355712/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775653.179738:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775653.179740:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775653.179744:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710355712/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775653.179752:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775653.179755:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775653.179761:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775653.179765:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775653.179768:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775653.179774:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12545) 00000800:00000200:1.1:1775775653.180043:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775653.180046:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775775653.180456:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775653.180459:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775775653.180776:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775653.180779:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00010000:00000001:3.0:1775775653.307266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775653.307274:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775653.307276:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775653.307281:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775653.307286:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775653.307288:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775653.307290:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775653.307292:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775653.307295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775653.307297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775653.307300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775653.307300:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775653.307365:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775653.307372:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775653.307378:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775654.331317:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775654.331325:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775654.331332:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775654.331334:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775654.331338:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775654.331340:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775655.355241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775655.355246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775655.355249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775655.355250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775655.355253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775655.355254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775655.355296:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775655.355300:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775655.355303:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775656.379284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775656.379291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775656.379297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775656.379298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775656.379302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775656.379304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775657.083245:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307212038, at 4307212096 00000100:00000010:0.0:1775775657.083266:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2600. 00010000:00000001:3.0:1775775657.403236:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775657.403241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775657.403244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775657.403247:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775657.403249:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775657.403251:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775657.403254:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775657.403256:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775657.403257:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775657.403259:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775657.403262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775657.403263:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775657.403264:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775657.403264:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775657.403265:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775657.403266:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775657.403267:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775657.403268:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775657.403287:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775657.403289:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775657.403291:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775658.299256:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775658.299264:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775658.299268:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775658.299273:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775658.299277:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775658.299287:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12540 00000400:00000200:3.0:1775775658.299298:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775658.299304:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775658.299307:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775658.299316:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775658.299319:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775658.299321:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775658.299323:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775658.299326:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775658.299328:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775658.299331:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775658.299333:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775658.299335:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775658.299337:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775658.299340:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775658.299342:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775658.299343:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775658.299345:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775658.299348:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775658.299350:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775658.299352:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775658.299355:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775658.299357:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775658.299360:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775658.299362:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775658.299363:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775658.299373:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775775658.299376:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775658.299380:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775658.299381:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775658.299383:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775658.299386:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775658.299391:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775658.299399:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:3.0:1775775658.299406:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775658.299415:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710355840/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775658.299425:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775658.299429:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775658.299433:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710355840/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775658.299452:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775658.299455:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775658.299458:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775658.299459:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775658.299461:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775658.299463:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775658.299466:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12540 00000400:00000200:3.0:1775775658.299469:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775658.299493:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775658.299500:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775658.299510:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775658.299512:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775658.299522:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775658.299531:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775658.299560:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775658.299564:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775658.299566:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775658.299572:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775658.299579:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775658.299585:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775658.299595:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775658.299604:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775658.299625:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775658.299626:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775658.299628:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775658.299630:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775658.299632:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775658.299635:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775658.299646:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775658.299649:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775658.299650:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775658.299655:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775658.299658:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775658.299662:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775658.299674:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000020:00000040:3.0:1775775658.299677:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775658.299684:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775658.299688:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775658.299692:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775658.299694:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775658.299696:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775658.299701:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775775658.299706:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775658.299711:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710355968/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775658.299720:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775658.299727:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775658.299737:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710355968/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775658.299777:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775658.299784:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775658.299798:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775658.299804:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775658.299810:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775658.299816:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775658.299826:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12540 00000400:00000200:3.0:1775775658.299841:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775658.299846:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775658.299851:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775658.299859:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775658.299861:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775658.299864:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775658.299866:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775658.299871:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775658.299873:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775658.299876:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775658.299880:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775658.299882:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775658.299884:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775658.299887:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775658.299890:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775658.299896:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775658.299901:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775658.299903:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775658.299907:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775658.299915:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775658.299918:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775658.299922:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775658.299928:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775658.299930:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775658.299932:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775658.299936:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775658.299939:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775658.299943:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775775658.299948:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775658.299955:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775658.299956:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775658.299959:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775658.299963:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775658.299966:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775658.299972:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000001:3.0:1775775658.299978:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775658.299989:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710356096/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775658.299995:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775658.299997:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775658.300001:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710356096/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775658.300007:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775658.300011:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775658.300016:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775658.300021:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775658.300027:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775658.300035:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12551) 00000800:00000200:1.1:1775775658.300084:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775658.300088:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1775775658.300662:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775658.300665:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1775775658.300880:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775658.300882:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00010000:00000001:3.0:1775775658.427295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775658.427303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775658.427321:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775658.427322:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775658.427326:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775658.427328:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775658.427396:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775658.427400:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775658.427403:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775659.451245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775659.451250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775659.451255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775659.451256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775659.451259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775659.451260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775660.475273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775660.475278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775660.475283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775660.475285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775660.475289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775660.475290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775660.475356:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775660.475362:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775660.475365:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775661.499241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775661.499248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775661.499254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775661.499256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775661.499260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775661.499262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775662.203238:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307217159, at 4307217216 00000100:00000010:0.0:1775775662.203253:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2420. 00010000:00000001:3.0:1775775662.523233:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775662.523238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775662.523242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775662.523243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775662.523246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775662.523247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775662.523267:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775662.523269:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775662.523271:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775663.419275:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775663.419287:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775663.419289:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775775663.419291:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775775663.419295:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775663.419297:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775663.419300:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775663.419306:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00100000:2.0:1775775663.419307:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775663.419312:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775663.419314:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775775663.419321:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12546 00000100:00000001:2.0:1775775663.419322:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775663.419325:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775663.419327:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775775663.419336:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775663.419345:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775663.419349:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775663.419352:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775663.419355:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775663.419357:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775663.419359:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775663.419362:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775663.419364:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775663.419367:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775663.419370:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775663.419372:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775663.419373:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775663.419375:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775663.419377:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775663.419378:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775663.419379:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775663.419381:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775663.419384:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775663.419387:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775663.419391:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775663.419396:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775663.419399:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775663.419402:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775663.419405:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775663.419418:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73100. 00000020:00000040:3.0:1775775663.419423:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775663.419428:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775663.419430:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775663.419435:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775663.419438:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775663.419445:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775663.419456:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000001:3.0:1775775663.419465:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775663.419477:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73100 x1862022710356224/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775663.419490:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775663.419496:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775663.419502:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f67e6552] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73100 x1862022710356224/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775663.419530:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775663.419534:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775663.419539:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775663.419540:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775663.419542:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775663.419546:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775663.419549:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12546 00000400:00000200:3.0:1775775663.419554:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775663.419557:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775663.419560:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775663.419564:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775663.419566:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775663.419568:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775663.419571:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775663.419573:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775663.419575:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775663.419577:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775663.419579:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775663.419581:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775663.419584:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775663.419587:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775663.419589:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775663.419591:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775663.419595:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775663.419598:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775663.419603:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775663.419607:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775663.419609:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775663.419618:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775663.419622:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775663.419626:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775663.419633:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775663.419638:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775663.419644:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775663.419655:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775775663.419662:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775663.419672:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775663.419675:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775663.419679:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775663.419684:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775663.419687:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775663.419698:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:3.0:1775775663.419708:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775663.419723:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710356352/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775663.419744:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775663.419746:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775663.419756:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710356352/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775663.419788:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775663.419794:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775663.419807:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775663.419810:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775663.419814:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775663.419826:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775663.419836:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12546 00000400:00000200:3.0:1775775663.419854:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775663.419867:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775663.419876:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775663.419886:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775663.419891:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775663.419894:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775663.419897:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775663.419904:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775663.419908:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775663.419913:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775663.419918:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775663.419925:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775663.419931:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775663.419942:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775663.419949:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775663.419958:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775663.419970:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775663.419978:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775663.419985:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775663.419988:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775663.419993:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775663.420002:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775663.420013:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775663.420032:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775663.420035:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775663.420041:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775663.420047:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775663.420053:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000020:00000040:3.0:1775775663.420058:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775663.420064:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775663.420068:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775663.420073:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775663.420076:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775663.420080:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775663.420087:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000800:00000200:1.1:1775775663.420089:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775775663.420094:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775663.420095:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000100:00100000:3.0:1775775663.420106:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710356480/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775663.420129:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775663.420177:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775663.420186:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710356480/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775663.420236:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775663.420241:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775663.420251:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775663.420259:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775663.420279:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775663.420289:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12556) 00000800:00000200:1.1:1775775663.420654:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775663.420656:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1775775663.420886:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775663.420888:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00010000:00000001:3.0:1775775663.547261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775663.547270:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775663.547272:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775663.547276:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775663.547281:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775663.547283:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775663.547285:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775663.547287:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775663.547291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775663.547292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775663.547295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775663.547296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775663.547381:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775663.547388:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775663.547393:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775664.443199:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775664.443206:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775664.443207:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775664.443212:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775664.443214:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775664.443217:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775664.443220:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775664.443221:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775664.443223:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775664.571303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775664.571309:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775664.571314:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775664.571315:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775664.571319:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775664.571320:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775665.467270:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775665.467277:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775665.467279:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775665.467283:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775665.467286:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775665.467289:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775665.467291:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775665.467293:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775665.467295:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775665.595212:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775665.595218:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775665.595225:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775665.595226:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775665.595230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775665.595231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775665.595291:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775665.595294:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775665.595296:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775666.491219:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775666.491224:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775666.491226:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775666.491232:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775666.491236:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775666.491239:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775666.491242:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775666.491244:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775666.491246:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775666.619232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775666.619237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775666.619241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775666.619242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775666.619245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775666.619246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775667.323226:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307222279, at 4307222336 00000100:00000010:0.0:1775775667.323238:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2360. 00000100:00000001:2.0:1775775667.515219:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775667.515225:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775667.515227:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775667.515232:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775667.515234:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775667.515237:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775667.515240:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775667.515242:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775667.515244:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775667.643227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775667.643233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775667.643237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775667.643240:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775667.643242:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775667.643244:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775667.643246:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775667.643248:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775667.643250:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775667.643251:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775667.643254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775667.643255:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775667.643256:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775667.643257:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775667.643258:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775667.643258:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775667.643259:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775667.643261:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775667.643278:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775667.643280:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775667.643282:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775668.539216:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775775668.539218:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775668.539223:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775668.539225:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775668.539225:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775668.539228:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1775775668.539229:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:3.0:1775775668.539231:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775668.539233:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775668.539234:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775775668.539236:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775668.539239:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775668.539241:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775775668.539241:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12551 00000100:00000001:2.0:1775775668.539244:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775775668.539250:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775668.539257:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775668.539260:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775668.539264:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775668.539267:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775668.539270:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775668.539272:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775668.539276:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775668.539278:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775668.539280:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775668.539283:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775668.539293:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775668.539295:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775668.539297:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775668.539299:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775668.539300:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775668.539302:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775668.539304:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775668.539306:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775668.539308:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775668.539311:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775668.539313:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775668.539315:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775668.539317:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775668.539318:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775668.539323:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775775668.539326:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775668.539329:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775668.539330:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775668.539333:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775668.539335:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775668.539338:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775668.539344:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000100:00000001:3.0:1775775668.539349:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775668.539357:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710356608/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775668.539365:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775668.539368:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775668.539371:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710356608/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775668.539387:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775668.539389:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775668.539392:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775668.539393:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775668.539394:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775668.539396:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775668.539397:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12551 00000400:00000200:3.0:1775775668.539400:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775668.539402:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775668.539404:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775668.539405:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775668.539407:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775668.539408:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775668.539410:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775668.539411:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775668.539412:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775668.539413:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775668.539414:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775668.539415:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775668.539417:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775668.539418:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775668.539420:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775668.539422:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775668.539423:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775668.539424:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775668.539427:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775668.539428:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775668.539430:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775668.539434:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775668.539437:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775668.539438:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775668.539443:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775668.539447:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775668.539450:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775668.539453:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775775668.539458:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775668.539464:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775668.539466:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775668.539469:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775668.539471:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775668.539474:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775668.539477:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000100:00000001:3.0:1775775668.539480:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775668.539487:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710356736/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775668.539499:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775668.539502:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775668.539507:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710356736/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775668.539521:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775668.539523:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775668.539528:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775668.539531:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775668.539533:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775668.539535:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775668.539538:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12551 00000400:00000200:3.0:1775775668.539543:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775668.539546:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775668.539547:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775668.539552:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775668.539554:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775668.539555:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775668.539558:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775668.539561:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775668.539563:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775668.539564:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775668.539566:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775668.539567:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775668.539569:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775668.539574:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775668.539576:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775668.539579:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775668.539584:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775668.539586:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775668.539590:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775668.539592:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775668.539595:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775668.539597:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775668.539602:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775668.539606:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775668.539608:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775668.539613:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775668.539614:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775668.539617:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775775668.539620:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775668.539624:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775668.539625:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775668.539627:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775668.539628:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775668.539631:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775668.539635:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000100:00000001:3.0:1775775668.539639:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775668.539646:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710356864/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775668.539661:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775668.539665:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775668.539670:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710356864/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775668.539683:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775668.539686:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775668.539690:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775668.539694:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775668.539697:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775668.539703:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12561) 00000800:00000200:1.1:1775775668.539764:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775668.539767:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775775668.540067:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775668.540069:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775775668.540277:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775668.540278:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00010000:00000001:3.0:1775775668.667204:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775668.667210:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775668.667221:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775668.667222:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775668.667225:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775668.667226:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775668.667285:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775668.667288:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775668.667291:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775669.563196:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775669.563202:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775669.563204:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775669.563209:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775669.563214:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775669.563217:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775669.563220:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775669.563222:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775669.563225:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775669.691231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775669.691236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775669.691240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775669.691241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775669.691244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775669.691245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775670.587239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775670.587246:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775670.587248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775670.587262:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775670.587266:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775670.587270:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775670.587273:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775670.587276:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775670.587279:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775670.715213:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775670.715220:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775670.715235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775670.715237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775670.715241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775670.715242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775670.715314:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775670.715318:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775670.715321:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775671.739248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775671.739253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775671.739257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775671.739259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775671.739262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775671.739263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775672.443224:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307227398, at 4307227456 00000100:00000010:0.0:1775775672.443244:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e22a0. 00010000:00000001:3.0:1775775672.763243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775672.763248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775672.763252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775672.763253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775672.763256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775672.763257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775672.763280:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775672.763283:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775672.763285:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775673.659229:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775673.659239:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775673.659242:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775673.659247:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775673.659254:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775673.659262:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12556 00000400:00000200:3.0:1775775673.659274:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775673.659280:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775673.659283:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775673.659288:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775673.659291:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775673.659294:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775673.659297:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775673.659301:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775673.659303:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775673.659306:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775673.659309:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775673.659311:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775673.659313:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775673.659317:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775673.659319:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775673.659322:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775673.659324:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775673.659327:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775673.659331:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775673.659334:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775673.659337:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775673.659339:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775673.659342:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775673.659344:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775673.659346:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775673.659355:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775775673.659359:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775673.659363:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775673.659364:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775673.659367:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775673.659370:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775673.659374:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775673.659382:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000100:00000001:3.0:1775775673.659388:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775673.659397:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710356992/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775673.659408:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775673.659413:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775673.659418:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710356992/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775673.659441:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775673.659445:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775673.659448:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775673.659450:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775673.659452:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775673.659454:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775673.659457:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12556 00000400:00000200:3.0:1775775673.659461:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775673.659464:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775673.659466:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775673.659469:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775673.659471:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775673.659473:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775673.659476:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775673.659478:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775673.659480:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775673.659481:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775673.659483:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775673.659485:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775673.659487:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775673.659491:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775673.659493:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775673.659496:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775673.659499:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775673.659501:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775673.659504:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775673.659507:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775673.659509:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775673.659516:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775673.659520:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775673.659523:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775673.659528:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775673.659534:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775673.659538:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775673.659544:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000020:00000040:3.0:1775775673.659548:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775673.659553:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775673.659557:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775673.659560:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775673.659562:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775673.659565:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775673.659570:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000100:00000001:3.0:1775775673.659577:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775673.659586:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710357120/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775673.659598:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775673.659603:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775673.659612:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710357120/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775673.659627:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775673.659633:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775673.659639:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775673.659643:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775673.659646:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775673.659651:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775673.659657:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12556 00000400:00000200:3.0:1775775673.659665:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775673.659672:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775673.659675:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775673.659682:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775673.659685:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775673.659692:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775673.659695:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775673.659700:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775673.659702:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775673.659706:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775673.659710:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775673.659715:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775673.659720:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775673.659727:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775673.659732:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775673.659735:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775673.659739:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775673.659745:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775673.659752:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775673.659757:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775673.659760:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775673.659765:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775673.659770:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775673.659775:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775673.659778:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775673.659783:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775673.659783:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775775673.659787:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775673.659788:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000100:00000010:3.0:1775775673.659793:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775775673.659797:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775673.659801:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775673.659803:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775673.659807:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775673.659810:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775673.659813:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775673.659819:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000100:00000001:3.0:1775775673.659824:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775673.659833:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710357248/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775673.659864:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775673.659869:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775673.659879:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710357248/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775673.659894:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775673.659900:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775673.659907:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775673.659911:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775673.659916:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775673.659922:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12566) 00000800:00000200:1.1:1775775673.660235:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775673.660236:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775775673.660540:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775673.660543:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00010000:00000001:3.0:1775775673.787241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775673.787250:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775673.787252:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775673.787256:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775673.787261:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775673.787263:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775673.787265:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775673.787268:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775673.787272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775673.787273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775673.787278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775673.787279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775673.787381:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775673.787387:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775673.787391:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775674.811230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775674.811236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775674.811242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775674.811244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775674.811250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775674.811251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775675.835226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775675.835233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775675.835239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775675.835240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775675.835243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775675.835244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775675.835309:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775675.835315:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775675.835319:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775676.859202:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775676.859209:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775676.859215:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775676.859217:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775676.859221:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775676.859224:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775677.563235:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307232518, at 4307232576 00000100:00000010:0.0:1775775677.563246:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e29c0. 00010000:00000001:3.0:1775775677.883257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775677.883264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775677.883271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775677.883276:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775677.883278:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775677.883283:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775677.883286:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775677.883290:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775677.883292:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775677.883295:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775677.883300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775677.883301:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775677.883303:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775677.883304:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775677.883306:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775677.883308:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775677.883310:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775677.883312:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775677.883341:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775677.883345:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775677.883349:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775678.779264:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775678.779271:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775678.779274:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775678.779279:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775678.779286:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775678.779295:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12561 00000400:00000200:0.0:1775775678.779305:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775678.779312:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775678.779315:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775678.779318:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775678.779321:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775678.779323:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775678.779326:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775678.779329:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775678.779330:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775678.779333:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775678.779335:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775678.779337:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775678.779339:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775678.779341:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775678.779343:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775678.779344:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775678.779345:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775678.779348:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775678.779350:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775678.779351:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775678.779355:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775678.779358:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775678.779361:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775678.779362:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775678.779365:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775678.779374:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061180. 00000020:00000040:0.0:1775775678.779377:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775678.779382:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775678.779383:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775678.779387:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775678.779390:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775678.779393:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775678.779401:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7800. 00000100:00000001:0.0:1775775678.779407:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775678.779416:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061180 x1862022710357376/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775678.779425:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775678.779429:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775678.779433:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e23859] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061180 x1862022710357376/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775678.779450:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775678.779453:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775678.779456:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775678.779457:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775678.779459:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775678.779461:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775678.779464:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12561 00000400:00000200:0.0:1775775678.779467:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775678.779469:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775678.779471:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775678.779475:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775678.779478:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775678.779481:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775678.779483:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775678.779486:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775678.779487:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775678.779490:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775678.779493:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775678.779497:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775678.779502:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775678.779518:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775678.779521:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775678.779526:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775678.779527:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775678.779533:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775678.779536:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775678.779539:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775678.779541:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775678.779547:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775678.779549:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775678.779552:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775678.779556:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775678.779560:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775678.779563:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775678.779566:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061880. 00000020:00000040:0.0:1775775678.779570:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775678.779573:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775678.779576:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775678.779578:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775678.779579:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775678.779580:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775678.779584:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6000. 00000100:00000001:0.0:1775775678.779588:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775678.779592:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061880 x1862022710357504/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775678.779604:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775678.779606:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775678.779618:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005afd1f27] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061880 x1862022710357504/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775678.779646:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775678.779651:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775678.779657:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775678.779661:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775678.779663:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775678.779669:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775678.779675:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12561 00000400:00000200:0.0:1775775678.779723:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775678.779731:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775678.779734:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775678.779739:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775678.779741:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775678.779743:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775678.779745:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775678.779748:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775678.779771:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775678.779773:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775678.779776:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775678.779779:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775678.779781:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:1.1:1775775678.779783:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:0.0:1775775678.779787:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775678.779790:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1775775678.779791:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000100:00000040:0.0:1775775678.779792:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775678.779796:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775678.779803:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775678.779809:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775678.779812:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775678.779814:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775678.779817:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775678.779821:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775678.779824:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775678.779828:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775678.779831:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775678.779836:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775678.779840:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060700. 00000020:00000040:0.0:1775775678.779843:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775678.779845:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775678.779846:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775678.779850:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775678.779851:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775678.779855:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775678.779858:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000001:0.0:1775775678.779864:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775678.779872:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060700 x1862022710357632/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775678.779879:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775678.779882:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775678.779888:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000225c8695] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060700 x1862022710357632/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775678.779911:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775678.779916:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775678.779920:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775678.779922:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775678.779925:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775678.779931:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12571) 00000800:00000200:1.1:1775775678.780209:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775678.780211:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775775678.780405:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775678.780407:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00010000:00000001:3.0:1775775678.907244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775678.907249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775678.907253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775678.907254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775678.907257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775678.907258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775678.907328:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775678.907335:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775678.907340:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775679.931228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775679.931233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775679.931237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775679.931238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775679.931241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775679.931242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775680.955249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775680.955258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775680.955265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775680.955267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775680.955285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775680.955287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775680.955337:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775680.955341:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775680.955344:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775681.979210:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775681.979215:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775681.979219:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775681.979220:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775681.979223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775681.979224:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775682.683283:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307237638, at 4307237696 00000100:00000010:0.0:1775775682.683297:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e21e0. 00000800:00000200:2.0:1775775682.683326:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775775682.683334:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775775682.683340:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1775775682.683519:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775775682.683533:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775775682.683538:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775775682.683540:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1775775683.003258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775683.003265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775683.003269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775683.003270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775683.003273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775683.003274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775683.003306:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775683.003309:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775683.003311:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775683.899247:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775683.899253:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775683.899255:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775683.899258:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775683.899261:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775683.899266:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12566 00000400:00000200:3.0:1775775683.899274:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775683.899279:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775683.899282:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775683.899284:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775683.899286:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775683.899288:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775683.899289:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775683.899292:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775683.899294:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775683.899296:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775683.899298:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775683.899300:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775683.899301:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775683.899303:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775683.899305:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775683.899306:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775683.899307:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775683.899309:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775683.899311:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775683.899313:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775683.899315:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775683.899317:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775683.899319:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775683.899320:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775683.899322:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775683.899328:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775775683.899331:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775683.899334:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775683.899335:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775683.899337:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775683.899339:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775683.899342:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775683.899349:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000100:00000001:3.0:1775775683.899354:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775683.899361:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710357760/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775683.899368:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775683.899372:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775683.899375:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710357760/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775683.899391:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775683.899393:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775683.899396:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775683.899397:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775683.899398:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775683.899400:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775683.899402:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12566 00000400:00000200:3.0:1775775683.899405:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775683.899407:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775683.899409:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775683.899411:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775683.899413:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775683.899414:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775683.899417:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775683.899420:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775683.899421:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775683.899424:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775683.899427:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775683.899429:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775683.899431:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775683.899436:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775683.899441:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775683.899445:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775683.899449:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775683.899452:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775683.899457:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775683.899462:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775683.899466:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775683.899474:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775683.899479:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775683.899482:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775683.899486:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775683.899491:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775683.899492:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775683.899495:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775775683.899499:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775683.899501:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775683.899503:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775683.899506:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775683.899509:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775683.899512:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775683.899519:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:3.0:1775775683.899522:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775683.899529:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710357888/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775683.899539:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775683.899541:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775683.899549:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710357888/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775683.899555:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775683.899557:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775683.899560:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775683.899563:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775683.899565:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775683.899567:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775683.899571:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12566 00000400:00000200:3.0:1775775683.899580:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775683.899584:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775683.899587:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775683.899591:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775683.899593:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775683.899598:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775683.899601:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775683.899602:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775683.899604:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775683.899606:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775683.899609:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775683.899612:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775683.899614:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775683.899618:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775683.899623:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775683.899624:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775683.899627:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775683.899630:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775683.899635:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775683.899637:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775683.899640:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775683.899642:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775683.899645:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775683.899649:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775683.899652:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775683.899655:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775683.899657:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775683.899675:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775775683.899677:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775683.899678:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775683.899679:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775683.899680:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775683.899681:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775683.899683:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775683.899685:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000001:3.0:1775775683.899688:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775683.899694:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710358016/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775683.899709:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775683.899713:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775683.899720:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710358016/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775683.899736:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775683.899741:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775683.899746:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775683.899748:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775683.899752:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775683.899756:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12576) 00000800:00000200:1.1:1775775683.899767:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775683.899770:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775775683.900086:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775683.900087:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775775683.900323:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775683.900325:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00010000:00000001:3.0:1775775684.027257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775684.027265:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775684.027267:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775684.027270:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775684.027274:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775684.027276:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775684.027277:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775684.027279:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775684.027282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775684.027283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775684.027286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775684.027287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775684.027350:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775684.027353:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775684.027355:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775685.051216:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775685.051222:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775685.051228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775685.051230:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775685.051234:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775685.051235:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775685.947294:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775685.947300:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775685.947301:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775685.947307:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775685.947309:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775685.947312:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775685.947314:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775685.947316:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775685.947318:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775686.075235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775686.075240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775686.075246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775686.075248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775686.075252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775686.075254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775686.075303:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775686.075306:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775686.075310:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775686.971214:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775686.971221:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775686.971223:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775686.971228:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775686.971232:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775686.971236:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775686.971238:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775686.971240:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775686.971243:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775687.099220:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775687.099225:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775687.099229:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775687.099230:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775687.099233:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775687.099234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775687.803214:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307242758, at 4307242816 00000100:00000010:0.0:1775775687.803227:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2cc0. 00000100:00000001:2.0:1775775687.995228:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775687.995234:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775687.995237:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775687.995243:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775687.995247:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775687.995251:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775687.995254:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775687.995256:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775687.995259:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775688.123222:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775688.123228:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775688.123232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775688.123234:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775688.123236:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775688.123239:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775688.123241:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775688.123243:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775688.123244:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775688.123246:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775688.123249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775688.123250:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775688.123250:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775688.123251:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775688.123252:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775688.123253:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775688.123254:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775688.123255:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775688.123277:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775688.123279:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775688.123281:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775689.019249:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775775689.019249:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775775689.019255:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775689.019259:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775689.019260:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1775775689.019265:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775689.019266:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775775689.019271:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775775689.019272:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775689.019274:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775689.019279:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775689.019279:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775775689.019281:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775689.019285:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775689.019289:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12571 00000400:00000200:3.0:1775775689.019301:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775689.019309:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775689.019313:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775689.019318:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775689.019321:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775689.019324:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775689.019329:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775689.019333:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775689.019335:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775689.019339:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775689.019342:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775689.019344:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775689.019347:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775689.019351:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775689.019353:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775689.019355:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775689.019358:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775689.019361:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775689.019365:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775689.019367:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775689.019370:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775689.019374:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775689.019377:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775689.019379:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775689.019382:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775689.019391:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73b80. 00000020:00000040:3.0:1775775689.019395:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775689.019398:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775689.019400:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775689.019405:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775689.019408:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775689.019412:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775689.019421:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000001:3.0:1775775689.019430:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775689.019441:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73b80 x1862022710358144/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775689.019454:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775689.019459:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775689.019466:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c52008f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73b80 x1862022710358144/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775689.019495:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775689.019498:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775689.019503:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775689.019505:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775689.019507:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775689.019511:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775689.019515:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12571 00000400:00000200:3.0:1775775689.019519:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775689.019524:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775689.019528:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775689.019533:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775689.019536:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775689.019540:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775689.019545:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775689.019551:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775689.019554:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775689.019559:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775689.019561:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775689.019568:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775689.019588:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775689.019597:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775689.019621:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775689.019627:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775689.019630:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775689.019635:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775689.019644:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775689.019650:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775689.019654:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775689.019662:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775689.019666:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775689.019669:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775689.019675:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775689.019681:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775689.019684:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775689.019693:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775775689.019697:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775689.019700:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775689.019703:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775689.019706:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775689.019710:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775689.019714:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775689.019723:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775775689.019730:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775689.019743:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710358272/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775689.019762:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775689.019767:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775689.019775:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710358272/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775689.019795:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775689.019802:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775689.019811:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775689.019815:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775689.019818:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775689.019825:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775689.019831:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12571 00000400:00000200:3.0:1775775689.019841:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775689.019847:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775689.019853:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775689.019861:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775689.019866:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1775775689.019866:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775689.019870:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000100:00000040:3.0:1775775689.019871:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775689.019892:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775689.019898:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775689.019903:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775689.019906:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775689.019909:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775689.019916:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775689.019920:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775689.019929:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775689.019932:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775689.019939:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775689.019945:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775689.019952:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775689.019960:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775689.019968:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775689.019972:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775689.019978:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775689.019984:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775689.019990:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775689.019994:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775689.020000:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775689.020005:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775689.020011:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775775689.020015:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775689.020022:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775689.020026:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775689.020031:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775689.020032:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775689.020035:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775689.020045:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:3.0:1775775689.020050:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775689.020058:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710358400/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775689.020067:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775689.020073:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775689.020080:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710358400/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775689.020102:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775689.020110:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775689.020124:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775689.020131:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775689.020136:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775689.020143:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12581) 00000800:00000200:1.1:1775775689.020412:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775689.020414:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775775689.020710:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775689.020713:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00010000:00000001:3.0:1775775689.147263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775689.147269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775689.147273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775689.147275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775689.147278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775689.147279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775689.147379:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775689.147383:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775689.147386:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775690.043278:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775690.043284:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775690.043285:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775690.043290:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775690.043293:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775690.043296:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775690.043298:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775690.043300:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775690.043301:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775690.171278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775690.171288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775690.171299:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775690.171301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775690.171308:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775690.171310:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775691.067224:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775691.067230:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775691.067231:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775691.067236:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775691.067239:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775691.067242:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775691.067244:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775691.067247:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775691.067250:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775691.195253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775691.195257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775691.195262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775691.195262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775691.195265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775691.195266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775691.195321:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775691.195325:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775691.195328:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775692.219208:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775692.219212:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775692.219216:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775692.219217:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775692.219220:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775692.219221:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775692.923264:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307247878, at 4307247936 00000100:00000010:0.0:1775775692.923302:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2d20. 00010000:00000001:3.0:1775775693.243251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775693.243257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775693.243261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775693.243263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775693.243265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775693.243266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775693.243286:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775693.243288:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775693.243290:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775694.139241:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775694.139255:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775694.139257:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775694.139260:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775694.139269:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775694.139278:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12576 00000400:00000200:3.0:1775775694.139293:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775694.139303:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775694.139306:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775694.139309:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775694.139312:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775694.139314:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775694.139317:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775694.139321:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775694.139323:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775694.139325:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775694.139327:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775694.139329:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775694.139330:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775694.139333:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775694.139334:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775694.139335:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775694.139336:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775694.139339:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775694.139341:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775694.139342:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775694.139345:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775694.139347:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775694.139348:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775694.139350:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775694.139351:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775694.139359:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775775694.139361:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775694.139364:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775694.139365:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775694.139367:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775694.139370:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775694.139373:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775694.139381:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000001:3.0:1775775694.139386:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775694.139393:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710358528/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775694.139401:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775694.139405:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775694.139409:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710358528/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775694.139428:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775694.139431:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775694.139433:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775694.139435:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775694.139436:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775694.139438:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775694.139440:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12576 00000400:00000200:3.0:1775775694.139443:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775694.139446:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775694.139448:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775694.139451:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775694.139453:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775694.139454:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775694.139456:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775694.139459:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775694.139463:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775694.139465:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775694.139467:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775694.139471:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775694.139474:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775694.139477:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775694.139478:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775694.139501:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775694.139504:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775694.139510:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775694.139518:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775694.139523:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775694.139531:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775694.139540:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775694.139544:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775694.139547:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775694.139551:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775694.139554:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775694.139566:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775694.139572:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775775694.139579:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775694.139587:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775694.139592:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775694.139598:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775694.139599:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775694.139601:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775694.139608:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000100:00000001:3.0:1775775694.139615:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775694.139621:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710358656/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775694.139644:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775694.139649:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775694.139662:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710358656/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775694.139672:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775694.139675:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775694.139678:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775694.139679:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775694.139681:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775694.139683:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775694.139686:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12576 00000400:00000200:3.0:1775775694.139700:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775694.139706:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775694.139711:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775694.139718:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775694.139725:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775694.139730:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775694.139738:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775694.139744:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775694.139747:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775694.139752:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775694.139756:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775694.139759:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775694.139765:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775694.139771:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775694.139775:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775694.139779:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775694.139781:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775694.139785:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775694.139790:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775694.139795:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775694.139801:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775694.139805:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775694.139807:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775694.139808:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775694.139809:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775694.139815:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775694.139819:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775694.139826:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000020:00000040:3.0:1775775694.139828:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775694.139830:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775694.139831:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775694.139832:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775694.139833:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775694.139835:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775694.139841:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000001:3.0:1775775694.139845:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775694.139858:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710358784/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775694.139875:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775694.139883:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775694.139892:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710358784/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775694.139912:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775694.139922:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775694.139931:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775694.139933:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775694.139937:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775694.139941:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12586) 00000800:00000200:1.1:1775775694.139980:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775694.139983:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775775694.140345:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775694.140347:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775775694.140596:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775694.140598:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00010000:00000001:3.0:1775775694.267218:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775694.267225:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775694.267228:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775694.267231:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775694.267236:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775694.267238:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775694.267240:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775694.267242:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775694.267247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775694.267248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775694.267252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775694.267254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775694.267305:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775694.267309:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775694.267311:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775695.291298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775695.291306:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775695.291313:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775695.291315:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775695.291321:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775695.291323:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775695.291390:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775695.291394:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775695.291397:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775696.315218:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775696.315223:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775696.315227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775696.315228:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775696.315230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775696.315231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775696.315286:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775696.315290:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775696.315294:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775697.339233:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775697.339237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775697.339242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775697.339244:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775697.339246:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775697.339248:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775697.339251:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775697.339252:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775697.339254:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775697.339255:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775697.339258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775697.339259:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775697.339260:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775697.339261:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775697.339262:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775697.339263:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775697.339264:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775697.339265:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775698.043281:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307252998, at 4307253056 00000100:00000010:0.0:1775775698.043307:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ae0. 00010000:00000001:3.0:1775775698.363305:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775698.363312:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775698.363317:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775698.363319:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775698.363322:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775698.363323:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775698.363349:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775698.363351:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775698.363354:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775699.259240:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775699.259254:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775699.259258:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775699.259266:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775699.259276:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775699.259287:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12581 00000400:00000200:3.0:1775775699.259298:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775699.259305:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775699.259308:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775699.259311:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775699.259313:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775699.259315:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775699.259317:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775699.259320:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775699.259322:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775699.259324:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775699.259326:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775699.259328:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775699.259330:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775699.259332:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775699.259333:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775699.259335:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775699.259336:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775699.259338:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775699.259341:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775699.259343:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775699.259345:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775699.259347:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775699.259349:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775699.259351:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775699.259352:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775699.259360:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775775699.259363:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775699.259366:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775699.259367:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775699.259370:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775699.259373:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775699.259376:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775699.259384:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000100:00000001:3.0:1775775699.259389:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775699.259414:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710358912/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775699.259423:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775699.259427:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775699.259430:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710358912/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775699.259451:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775699.259455:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775699.259459:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775699.259461:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775699.259463:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775699.259467:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775699.259470:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12581 00000400:00000200:3.0:1775775699.259478:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775699.259483:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775699.259490:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775699.259500:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775699.259509:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775699.259520:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775699.259529:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775699.259534:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775699.259540:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775699.259543:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775699.259549:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775699.259556:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775699.259566:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775699.259578:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775699.259590:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775699.259598:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775699.259600:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775699.259607:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775699.259612:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775699.259615:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775699.259620:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775699.259625:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775699.259628:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775699.259630:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775699.259637:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775699.259645:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775699.259647:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775699.259650:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775775699.259654:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775699.259659:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775699.259663:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775699.259664:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775699.259667:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775699.259673:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775699.259699:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775775699.259705:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775699.259722:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710359040/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775699.259746:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775699.259769:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775699.259791:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710359040/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775699.259833:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775699.259852:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775699.259861:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775699.259865:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775699.259869:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775699.259873:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775699.259880:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12581 00000400:00000200:3.0:1775775699.259893:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775699.259898:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775699.259908:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775699.259915:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775699.259921:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775699.259932:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775699.259937:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775699.259945:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775699.259952:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775699.259956:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775699.259962:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775699.259967:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775699.259973:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775699.259989:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775699.259996:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775699.260000:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775699.260002:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775699.260013:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775699.260026:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775699.260037:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775699.260042:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775699.260051:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775699.260059:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775699.260066:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775699.260070:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000200:1.1:1775775699.260071:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775699.260078:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000008:00000001:3.0:1775775699.260082:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775699.260090:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775699.260096:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775775699.260099:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775699.260105:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775699.260111:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775699.260119:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775699.260121:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775699.260124:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775699.260131:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000100:00000001:3.0:1775775699.260135:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775699.260143:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710359168/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775699.260247:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775699.260254:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775699.260282:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710359168/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775699.260300:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775699.260305:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775699.260314:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775699.260317:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775699.260323:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775699.260328:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12591) 00000800:00000200:1.1:1775775699.260629:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775699.260632:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775775699.260994:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775699.260996:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00010000:00000001:3.0:1775775699.387283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775699.387289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775699.387295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775699.387296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775699.387300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775699.387301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775699.387497:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775699.387501:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775699.387503:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775700.411217:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775700.411223:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775700.411228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775700.411229:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775700.411232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775700.411233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775700.411329:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775700.411335:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775700.411339:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775701.435269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775701.435292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775701.435299:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775701.435301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775701.435306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775701.435307:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775701.435397:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775701.435418:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775701.435421:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775702.459335:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775702.459374:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775702.459382:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775702.459384:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775702.459399:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775702.459401:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775703.163254:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307258119, at 4307258176 00000100:00000010:0.0:1775775703.163268:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2c60. 00010000:00000001:3.0:1775775703.483284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775703.483294:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775703.483298:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775703.483303:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775703.483310:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775703.483314:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775703.483317:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775703.483373:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775703.483390:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775703.483391:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775703.483395:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775703.483396:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775703.483419:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775703.483422:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775703.483426:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775704.379277:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775704.379287:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775704.379290:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775704.379295:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775704.379300:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775704.379341:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12586 00000400:00000200:3.0:1775775704.379351:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775704.379363:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775704.379368:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775704.379376:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775704.379380:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775704.379389:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775704.379403:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775704.379408:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775704.379410:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775704.379418:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775704.379422:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775704.379427:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775704.379434:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775704.379439:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775704.379447:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775704.379450:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775704.379453:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775704.379456:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775704.379460:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775704.379463:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775704.379467:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775704.379471:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775704.379474:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775704.379477:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775704.379479:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775704.379488:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775775704.379492:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775704.379496:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775704.379497:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775704.379501:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775704.379503:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775704.379509:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775704.379517:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000100:00000001:3.0:1775775704.379524:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775704.379533:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710359296/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775704.379546:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775704.379550:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775704.379555:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710359296/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775704.379591:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775704.379594:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775704.379598:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775704.379600:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775704.379602:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775704.379606:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775704.379609:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12586 00000400:00000200:3.0:1775775704.379616:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775704.379623:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775704.379627:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775704.379633:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775704.379637:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775704.379641:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775704.379650:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775704.379666:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775704.379670:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775704.379674:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775704.379679:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775704.379686:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775704.379696:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775704.379709:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775704.379714:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775704.379720:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775704.379729:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775704.379732:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775704.379739:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775704.379744:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775704.379749:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775704.379757:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775704.379761:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775704.379767:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775704.379775:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775704.379782:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775704.379788:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775704.379800:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000020:00000040:3.0:1775775704.379805:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775704.379810:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775704.379814:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775704.379827:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775704.379833:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775704.379836:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775704.379849:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775775704.379858:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775704.379871:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710359424/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775704.379898:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775704.379904:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775704.379913:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710359424/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775704.379935:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775704.379943:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775704.379953:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775704.379958:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775704.379962:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775704.379971:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775704.379979:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12586 00000400:00000200:3.0:1775775704.379988:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775704.380000:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775704.380005:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775704.380008:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775704.380010:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775704.380012:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775704.380014:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775704.380017:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775704.380018:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775704.380020:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775704.380022:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775704.380025:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775704.380033:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775704.380037:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775704.380044:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775704.380047:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775704.380049:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775704.380054:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775704.380071:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775704.380094:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775704.380099:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775704.380102:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775704.380106:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775704.380111:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775704.380114:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775704.380118:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775704.380121:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775704.380127:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775775704.380132:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775704.380138:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775704.380184:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775704.380190:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775704.380195:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775704.380200:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775704.380207:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000100:00000001:3.0:1775775704.380224:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775704.380234:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710359552/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775704.380245:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775704.380248:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775704.380254:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710359552/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775704.380268:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775704.380283:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775704.380290:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775775704.380295:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775775704.380296:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775704.380303:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775775704.380320:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000100:00000040:3.0:1775775704.380323:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12597) 00000800:00000200:1.1:1775775704.380866:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775704.380869:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775775704.381340:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775704.381361:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00010000:00000001:3.0:1775775704.507376:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775704.507384:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775704.507392:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775704.507394:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775704.507400:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775704.507402:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775704.507497:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775704.507503:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775704.507508:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775705.531226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775705.531231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775705.531235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775705.531236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775705.531239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775705.531240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775705.531320:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775705.531323:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775705.531327:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775706.555214:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775706.555219:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775706.555223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775706.555224:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775706.555226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775706.555227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775706.555317:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775706.555322:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775706.555326:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775707.579291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775707.579309:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775707.579316:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775707.579320:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775707.579323:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775707.579327:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775707.579331:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775707.579334:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775707.579337:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775707.579340:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775707.579344:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775707.579346:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775707.579347:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775707.579349:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775707.579352:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775707.579353:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775707.579356:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775707.579358:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775708.283320:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307263239, at 4307263296 00000100:00000010:0.0:1775775708.283349:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2540. 00010000:00000001:3.0:1775775708.603315:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775708.603324:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775708.603332:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775708.603334:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775708.603338:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775708.603340:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775708.603374:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775708.603377:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775708.603381:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775709.499343:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775709.499356:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775709.499361:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775709.499367:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775709.499374:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775709.499385:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12592 00000400:00000200:3.0:1775775709.499398:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775709.499407:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775709.499412:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775709.499445:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775709.499450:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775709.499454:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775709.499457:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775709.499463:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775709.499466:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775709.499470:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775709.499474:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775709.499477:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775709.499479:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775709.499483:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775709.499485:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775709.499488:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775709.499490:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775709.499494:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775709.499498:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775709.499501:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775709.499505:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775709.499509:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775709.499513:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775709.499516:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775709.499518:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775709.499534:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73100. 00000020:00000040:3.0:1775775709.499538:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775709.499545:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775709.499547:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775709.499551:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775709.499555:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775709.499562:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775709.499575:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000100:00000001:3.0:1775775709.499584:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775709.499597:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73100 x1862022710359680/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775709.499612:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775709.499619:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775709.499625:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f67e6552] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73100 x1862022710359680/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775709.499655:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775709.499660:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775709.499665:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775709.499667:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775709.499669:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775709.499673:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775709.499678:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12592 00000400:00000200:3.0:1775775709.499687:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775709.499696:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775709.499710:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775709.499723:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775709.499737:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775709.499749:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775709.499760:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775709.499769:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775709.499774:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775709.499781:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775709.499791:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775709.499799:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775709.499809:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775709.499826:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775709.499831:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775709.499833:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775709.499836:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775709.499845:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775709.499855:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775709.499859:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775709.499866:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775709.499875:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775709.499895:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775709.499898:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775709.499906:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775709.499916:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775709.499920:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775709.499932:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775775709.499940:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775709.499954:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775709.499959:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775709.499970:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775709.499980:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775709.499984:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775709.499992:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000001:3.0:1775775709.499998:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775709.500010:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710359808/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775709.500037:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775709.500041:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775709.500049:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710359808/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775709.500126:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775709.500179:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775709.500194:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775709.500198:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775709.500207:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.1:1775775709.500208:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775709.500233:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000100:00000001:3.0:1775775709.500237:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775709.500248:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12592 00000400:00000200:3.0:1775775709.500263:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775709.500271:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775709.500277:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775709.500282:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775709.500288:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775709.500295:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775709.500299:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775709.500308:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775709.500312:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775709.500316:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775709.500321:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775709.500328:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775709.500332:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775709.500343:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775709.500348:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775709.500355:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775709.500359:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775709.500369:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775709.500387:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775709.500399:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775709.500411:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775709.500419:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775709.500427:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775709.500437:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775709.500442:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775709.500457:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775709.500465:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775709.500483:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000020:00000040:3.0:1775775709.500491:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775709.500503:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775709.500505:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775709.500516:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775709.500519:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775709.500526:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775709.500536:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000001:3.0:1775775709.500545:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775709.500555:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710359936/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775709.500590:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775709.500596:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775709.500601:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710359936/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775709.500637:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775709.500642:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775709.500648:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775709.500654:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775709.500662:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775709.500670:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12602) 00000800:00000200:1.1:1775775709.500962:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775709.500964:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775775709.501251:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775709.501254:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00010000:00000001:3.0:1775775709.627309:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775709.627315:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775709.627321:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775709.627322:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775709.627325:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775709.627326:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775709.627382:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775709.627387:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775709.627391:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775710.651260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775710.651266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775710.651271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775710.651272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775710.651275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775710.651276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775710.651331:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775710.651335:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775710.651338:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775711.675260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775711.675264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775711.675268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775711.675269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775711.675272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775711.675273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775711.675319:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775711.675323:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775711.675326:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775712.699321:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775712.699327:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775712.699333:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775712.699335:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775712.699339:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775712.699340:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775713.403248:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307268359, at 4307268416 00000100:00000010:0.0:1775775713.403264:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2c60. 00010000:00000001:3.0:1775775713.723231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775713.723238:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775713.723240:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775713.723243:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775713.723246:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775713.723248:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775713.723249:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775713.723251:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775713.723254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775713.723255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775713.723258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775713.723259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775713.723275:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775713.723277:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775713.723279:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775714.619261:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775714.619268:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775714.619270:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775714.619273:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775714.619276:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775714.619283:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12597 00000400:00000200:3.0:1775775714.619290:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775714.619296:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775714.619298:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775714.619302:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775714.619305:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775714.619307:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775714.619309:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775714.619312:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775714.619314:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775714.619316:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775714.619318:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775714.619320:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775714.619321:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775714.619324:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775714.619325:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775714.619327:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775714.619328:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775714.619330:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775714.619332:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775714.619334:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775714.619336:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775714.619338:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775714.619340:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775714.619342:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775714.619343:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775714.619351:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775775714.619354:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775714.619358:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775714.619359:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775714.619361:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775714.619363:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775714.619367:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775714.619373:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775775714.619379:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775714.619386:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710360064/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775714.619394:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775714.619398:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775714.619401:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710360064/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775714.619415:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775714.619418:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775714.619420:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775714.619421:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775714.619422:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775714.619424:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775714.619426:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12597 00000400:00000200:3.0:1775775714.619428:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775714.619430:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775714.619432:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775714.619435:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775714.619436:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775714.619438:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775714.619439:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775714.619441:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775714.619444:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775714.619446:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775714.619449:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775714.619453:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775714.619455:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775714.619460:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775714.619465:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775714.619468:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775714.619473:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775714.619476:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775714.619480:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775714.619483:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775714.619486:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775714.619490:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775714.619494:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775714.619499:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775714.619503:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775714.619507:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775714.619511:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775714.619515:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775775714.619516:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775714.619518:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775714.619519:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775714.619522:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775714.619525:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775714.619527:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775714.619531:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:3.0:1775775714.619534:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775714.619540:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710360192/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775714.619550:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775714.619552:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775714.619559:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710360192/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775714.619598:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775714.619603:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775714.619608:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775714.619612:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775714.619614:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775714.619619:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775714.619623:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12597 00000400:00000200:3.0:1775775714.619629:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775714.619632:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775714.619634:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775714.619637:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775714.619640:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775714.619642:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775714.619645:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775714.619649:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775714.619650:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775714.619652:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775714.619655:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775714.619658:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775714.619661:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775714.619666:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775714.619669:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775714.619673:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775714.619677:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775714.619683:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775714.619689:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775714.619691:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775714.619695:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775714.619699:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775714.619702:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775714.619705:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775714.619706:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775714.619711:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775714.619714:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775714.619717:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775775714.619721:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775714.619722:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775714.619723:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775714.619726:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775714.619728:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775714.619731:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775714.619734:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000010:3.0:1775775714.619735:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000001:3.0:1775775714.619740:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775714.619745:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710360320/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775714.619750:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775714.619751:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775714.619754:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710360320/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775714.619765:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775714.619768:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775714.619772:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775714.619775:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775714.619779:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775714.619783:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12607) 00000800:00000200:1.1:1775775714.619941:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775714.620069:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00010000:00000001:3.0:1775775714.747250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775714.747257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775714.747263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775714.747264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775714.747268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775714.747270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775714.747336:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775714.747340:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775714.747344:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775715.771217:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775715.771222:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775715.771226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775715.771227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775715.771230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775715.771231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775715.771295:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775715.771299:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775715.771303:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775716.795230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775716.795238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775716.795244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775716.795245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775716.795248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775716.795250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775716.795278:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775716.795283:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775716.795286:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775775717.691515:0:2401:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:2.0:1775775717.691562:0:2401:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1775775717.691578:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775717.691585:0:2401:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:2.0:1775775717.691594:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775717.691600:0:2401:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:2.0:1775775717.691609:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775717.691616:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:2.0:1775775717.691625:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775717.691633:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775717.691660:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3088 00000400:00000010:2.0:1775775717.691664:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3088. 00000100:00000001:2.0:1775775717.691673:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775717.691685:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c73480 x1862022710360064/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775739 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775717.691731:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775717.691754:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73480 x1862022710360064/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775739 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775717.691767:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775717.691771:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775775717.691784:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775717.691788:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230c00. 00000400:00000200:2.0:1775775717.691792:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775717.691798:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775717.691803:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3330 00000400:00000010:2.0:1775775717.691806:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3330. 00000100:00000001:2.0:1775775717.691810:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775717.691817:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c70380 x1862022710360192/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775739 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775717.691826:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775717.691830:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70380 x1862022710360192/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775739 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775717.691841:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775717.691844:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775775717.691852:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775717.691855:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230000. 00000400:00000200:2.0:1775775717.691860:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775717.691865:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775717.691871:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3dd0 00000400:00000010:2.0:1775775717.691873:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3dd0. 00000100:00000001:2.0:1775775717.691876:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775717.691882:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c71c00 x1862022710360320/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775739 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775717.691892:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775717.691896:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71c00 x1862022710360320/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775739 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775717.691905:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775717.691907:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:3.0:1775775717.819244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775717.819251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775717.819256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775717.819260:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775717.819262:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775717.819266:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775717.819269:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775717.819271:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775717.819273:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775717.819276:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775717.819280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775717.819282:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775717.819283:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775717.819284:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775717.819286:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775717.819288:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775717.819290:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775717.819292:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775718.523222:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307273478, at 4307273536 00000100:00000010:0.0:1775775718.523236:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ae0. 00010000:00000001:3.0:1775775718.843254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775718.843260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775718.843266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775718.843267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775718.843272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775718.843273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775718.843300:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775718.843304:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775718.843307:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775719.739287:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775719.739295:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775719.739299:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775719.739303:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775719.739306:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775719.739313:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12602 00000400:00000200:0.0:1775775719.739322:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775719.739328:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775719.739331:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775719.739334:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775719.739337:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775719.739340:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775719.739342:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775719.739345:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775719.739347:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775719.739350:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775719.739352:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775719.739354:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775719.739356:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775719.739360:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775719.739362:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775719.739363:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775719.739365:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775719.739368:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775719.739370:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775719.739372:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775719.739375:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775719.739378:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775719.739380:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775719.739382:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775719.739384:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775719.739391:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061f80. 00000020:00000040:0.0:1775775719.739393:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775719.739397:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775719.739398:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775719.739401:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775719.739403:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775719.739407:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775719.739415:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000001:0.0:1775775719.739421:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775719.739445:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061f80 x1862022710360448/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775719.739458:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775719.739463:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775719.739470:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000029d3794] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061f80 x1862022710360448/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775719.739509:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775719.739513:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775719.739518:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775719.739520:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775719.739522:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775719.739526:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775719.739529:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12602 00000400:00000200:0.0:1775775719.739535:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775719.739539:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775719.739541:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775719.739545:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775719.739547:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775719.739550:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775719.739552:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775719.739555:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775719.739557:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775719.739559:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775719.739561:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775719.739563:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775719.739567:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775719.739570:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775719.739573:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775719.739575:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775719.739577:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775719.739580:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775719.739583:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775719.739586:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775719.739589:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775719.739596:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775719.739598:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775719.739600:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775719.739606:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775719.739610:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775719.739612:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775719.739618:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063b80. 00000020:00000040:0.0:1775775719.739621:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775719.739624:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775719.739625:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775719.739627:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775719.739629:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775719.739632:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775719.739636:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4400. 00000100:00000001:0.0:1775775719.739640:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775719.739647:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063b80 x1862022710360576/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775719.739655:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775719.739658:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775719.739663:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ee9007b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063b80 x1862022710360576/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775719.739674:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775719.739677:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775719.739682:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775719.739684:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775719.739686:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775719.739689:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775719.739693:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12602 00000400:00000200:0.0:1775775719.739697:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775719.739702:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775719.739705:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775719.739711:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775719.739717:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775719.739722:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775719.739726:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775719.739731:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775719.739736:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775719.739741:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775719.739745:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775719.739752:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775719.739757:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775719.739764:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775719.739769:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775719.739775:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775719.739783:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775719.739790:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775719.739798:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775719.739804:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775719.739808:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775719.739812:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775719.739820:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775719.739825:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775719.739829:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775719.739832:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775719.739839:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775719.739848:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061500. 00000020:00000040:0.0:1775775719.739854:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775719.739859:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775719.739861:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775719.739866:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775719.739868:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775719.739870:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775719.739875:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6000. 00000100:00000001:0.0:1775775719.739879:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775719.739887:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061500 x1862022710360704/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775719.739900:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775719.739903:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775719.739913:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb8d992] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061500 x1862022710360704/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775719.739931:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775719.739937:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775719.739947:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775719.739955:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775719.739962:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775719.739985:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12612) 00000800:00000200:1.1:1775775719.740157:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775719.740162:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775775719.740601:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775719.740604:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775775719.741093:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775719.741096:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00010000:00000001:3.0:1775775719.867214:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775719.867219:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775719.867223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775719.867224:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775719.867227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775719.867227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775719.867295:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775719.867299:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775719.867302:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.0:1775775720.571262:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775775720.571269:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775775720.571272:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00010000:00000001:3.0:1775775720.891276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775720.891285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775720.891293:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775720.891294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775720.891299:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775720.891300:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775720.891403:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775720.891409:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775720.891414:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775721.915223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775721.915230:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775721.915243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775721.915244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775721.915247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775721.915248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775721.915308:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775721.915313:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775721.915318:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775722.939226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775722.939231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775722.939235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775722.939236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775722.939239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775722.939240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775723.643250:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307278599, at 4307278656 00000100:00000010:0.0:1775775723.643269:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2d20. 00010000:00000001:3.0:1775775723.963199:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775723.963206:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775723.963208:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775723.963210:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775723.963214:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775723.963216:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775723.963218:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775723.963219:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775723.963222:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775723.963224:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775723.963226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775723.963227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775723.963244:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775723.963247:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775723.963249:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775724.859229:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775724.859238:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775724.859244:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775724.859250:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775724.859254:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775724.859260:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12607 00000400:00000200:3.0:1775775724.859269:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775724.859274:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775724.859277:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775724.859279:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775724.859282:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775724.859284:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775724.859286:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775724.859289:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775724.859290:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775724.859292:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775724.859294:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775724.859296:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775724.859298:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775724.859299:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775724.859301:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775724.859302:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775724.859304:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775724.859306:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775724.859308:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775724.859310:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775724.859313:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775724.859315:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775724.859317:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775724.859319:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775724.859321:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775724.859328:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775775724.859331:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775724.859334:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775724.859335:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775724.859338:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775724.859340:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775724.859343:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775724.859350:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000001:3.0:1775775724.859354:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775724.859361:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710360832/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775724.859369:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775724.859372:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775724.859376:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710360832/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775724.859395:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775724.859398:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775724.859402:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775724.859403:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775724.859406:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775724.859409:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775724.859413:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12607 00000400:00000200:3.0:1775775724.859418:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775724.859425:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775724.859430:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775724.859437:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775724.859444:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775724.859448:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775724.859452:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775724.859458:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775724.859459:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775724.859462:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775724.859465:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775724.859469:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775724.859473:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775724.859479:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775724.859483:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775724.859485:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775724.859489:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775724.859493:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775724.859497:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775724.859499:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775724.859500:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775724.859505:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775724.859508:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775724.859512:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775724.859518:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775724.859522:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775724.859523:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775724.859527:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000020:00000040:3.0:1775775724.859530:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775724.859532:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775724.859535:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775724.859538:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775724.859540:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775724.859543:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775724.859548:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:3.0:1775775724.859554:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775724.859562:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710360960/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775724.859574:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775724.859576:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775724.859582:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710360960/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775724.859601:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775724.859607:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775724.859616:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775724.859620:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775724.859623:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775724.859628:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775724.859634:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12607 00000400:00000200:3.0:1775775724.859641:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775724.859648:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775724.859651:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775724.859657:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775724.859659:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775724.859665:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775724.859670:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775724.859675:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775724.859677:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775724.859678:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775724.859682:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775724.859686:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775724.859688:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775724.859691:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775724.859693:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775724.859695:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775724.859698:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775724.859702:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775724.859706:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775724.859711:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775724.859712:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:1.1:1775775724.859715:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:3.0:1775775724.859717:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.1:1775775724.859718:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 02000000:00000001:3.0:1775775724.859720:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775724.859723:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775724.859726:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775724.859730:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775724.859733:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775724.859737:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775775724.859740:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775724.859744:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775724.859746:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775724.859748:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775724.859749:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775724.859751:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775724.859756:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000100:00000001:3.0:1775775724.859761:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775724.859765:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710361088/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775724.859774:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775724.859777:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775724.859783:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710361088/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775724.859797:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775724.859799:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775724.859802:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775724.859806:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775724.859814:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775724.859820:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12617) 00000800:00000200:1.1:1775775724.860091:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775724.860093:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775775724.860303:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775724.860305:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00010000:00000001:3.0:1775775724.987236:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775724.987241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775724.987246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775724.987247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775724.987250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775724.987251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775724.987295:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775724.987299:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775724.987301:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775726.011215:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775726.011222:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775726.011227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775726.011229:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775726.011232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775726.011234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775726.011294:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775726.011298:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775726.011301:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775727.035228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775727.035234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775727.035239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775727.035241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775727.035244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775727.035245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775727.035305:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775727.035309:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775727.035312:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775728.059240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775728.059245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775728.059249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775728.059251:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775728.059253:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775728.059256:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775728.059258:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775728.059260:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775728.059261:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775728.059263:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775728.059266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775728.059267:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775728.059267:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775728.059268:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775728.059269:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775728.059270:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775728.059272:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775728.059274:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775728.763246:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307283718, at 4307283776 00000100:00000010:0.0:1775775728.763261:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2cc0. 00010000:00000001:3.0:1775775729.083240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775729.083246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775729.083252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775729.083254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775729.083258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775729.083259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775729.083288:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775729.083290:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775729.083293:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775729.979260:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775729.979267:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775729.979271:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775729.979275:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775729.979280:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775729.979288:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12612 00000400:00000200:0.0:1775775729.979297:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775729.979305:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775729.979308:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775729.979312:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775729.979315:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775729.979318:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775729.979320:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775729.979324:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775729.979326:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775729.979329:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775729.979332:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775729.979334:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775729.979337:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775729.979340:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775729.979343:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775729.979345:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775729.979347:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775729.979350:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775729.979354:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775729.979357:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775729.979360:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775729.979363:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775729.979366:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775729.979369:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775729.979371:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775729.979380:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062a00. 00000020:00000040:0.0:1775775729.979384:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775729.979388:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775729.979390:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775729.979393:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775729.979396:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775729.979400:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775729.979408:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6400. 00000100:00000001:0.0:1775775729.979415:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775729.979424:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062a00 x1862022710361216/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775729.979443:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775729.979448:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775729.979453:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009753f9f6] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062a00 x1862022710361216/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775729.979473:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775729.979476:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775729.979480:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775729.979482:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775729.979484:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775729.979487:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775729.979491:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12612 00000400:00000200:0.0:1775775729.979496:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775729.979503:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775729.979506:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775729.979510:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775729.979514:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775729.979518:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775729.979523:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775729.979528:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775729.979530:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775729.979533:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775729.979537:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775729.979541:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775729.979546:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775729.979553:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775729.979558:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775729.979564:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775729.979569:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775729.979575:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775729.979582:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775729.979586:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775729.979591:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775729.979598:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775729.979605:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775729.979608:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775729.979614:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775729.979622:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775729.979626:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775729.979634:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062d80. 00000020:00000040:0.0:1775775729.979639:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775729.979645:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775729.979648:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775729.979653:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775729.979656:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775729.979661:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775729.979667:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5800. 00000100:00000001:0.0:1775775729.979675:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775729.979684:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062d80 x1862022710361344/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775729.979716:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775729.979723:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775729.979735:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009965f80b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062d80 x1862022710361344/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775729.979755:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775729.979760:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00080000:0.0:1775775729.979761:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775775729.979764:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000100:00080000:0.0:1775775729.979768:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775729.979773:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775729.979776:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775729.979780:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775729.979789:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12612 00000400:00000200:0.0:1775775729.979799:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775729.979806:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775729.979811:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775729.979817:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775729.979823:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775729.979829:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775729.979833:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775729.979837:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775729.979840:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775729.979844:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775729.979848:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775729.979852:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775729.979856:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775729.979891:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775729.979895:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775729.979899:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775729.979904:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775729.979909:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775729.979915:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775729.979921:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775729.979925:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775729.979930:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775729.979936:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775729.979940:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775729.979946:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775729.979949:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775729.979952:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775729.979956:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062680. 00000020:00000040:0.0:1775775729.979961:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775729.979967:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775729.979970:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775729.979974:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775729.979976:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775729.979979:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775729.979984:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6c00. 00000100:00000001:0.0:1775775729.979990:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775729.980000:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062680 x1862022710361472/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775729.980023:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775729.980027:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775729.980033:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000082c7985e] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062680 x1862022710361472/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775729.980050:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775729.980053:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775729.980061:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775729.980065:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775729.980069:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775729.980075:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12622) 00000800:00000200:1.1:1775775729.980234:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775729.980236:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775775729.980418:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775729.980419:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00010000:00000001:3.0:1775775730.107265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775730.107270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775730.107273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775730.107274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775730.107277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775730.107278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775730.107343:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775730.107347:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775730.107351:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775731.131209:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775731.131213:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775731.131217:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775731.131218:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775731.131220:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775731.131221:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775731.131269:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775731.131272:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775731.131274:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775732.155196:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775732.155201:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775732.155205:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775732.155206:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775732.155209:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775732.155210:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775732.155279:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775732.155282:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775732.155284:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775733.179207:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775733.179213:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775733.179219:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775733.179221:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775733.179226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775733.179226:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775733.883219:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307288838, at 4307288896 00000100:00000010:0.0:1775775733.883230:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e21e0. 00010000:00000001:3.0:1775775734.203229:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775734.203235:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775734.203237:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775734.203240:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775734.203243:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775734.203245:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775734.203247:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775734.203248:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775734.203252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775734.203253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775734.203257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775734.203258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775734.203282:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775734.203284:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775734.203288:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775735.099266:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775735.099274:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775735.099279:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775735.099282:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775735.099284:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775735.099290:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12617 00000400:00000200:3.0:1775775735.099298:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775735.099303:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775735.099305:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775735.099308:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775735.099310:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775735.099312:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775735.099313:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775735.099316:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775735.099318:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775735.099320:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775735.099322:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775735.099323:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775735.099325:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775735.099327:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775735.099328:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775735.099331:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775735.099337:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775735.099340:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775735.099344:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775735.099347:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775735.099350:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775735.099354:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775735.099357:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775735.099359:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775735.099362:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775735.099370:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775775735.099374:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775735.099378:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775735.099380:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775735.099382:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775735.099385:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775735.099389:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775735.099399:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000100:00000001:3.0:1775775735.099404:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775735.099411:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710361600/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775735.099419:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775735.099423:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775735.099426:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710361600/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775735.099443:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775735.099445:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775735.099448:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775735.099449:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775735.099450:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775735.099452:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775735.099454:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12617 00000400:00000200:3.0:1775775735.099457:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775735.099459:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775735.099461:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775735.099463:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775735.099465:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775735.099467:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775735.099469:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775735.099471:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775735.099472:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775735.099473:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775735.099474:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775735.099478:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775735.099480:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775735.099484:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775735.099487:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775735.099490:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775735.099492:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775735.099496:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775735.099502:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775735.099507:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775735.099511:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775735.099518:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775735.099520:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775735.099523:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775735.099528:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775735.099532:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775735.099537:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775735.099540:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775775735.099543:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775735.099547:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775735.099551:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775735.099553:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775735.099554:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775735.099556:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775735.099561:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000100:00000001:3.0:1775775735.099566:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775735.099571:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710361728/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775735.099577:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775735.099581:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775735.099583:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710361728/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775735.099596:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775735.099599:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775735.099604:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775735.099607:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775735.099610:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775735.099615:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775735.099619:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12617 00000400:00000200:3.0:1775775735.099626:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775735.099631:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775735.099633:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775735.099637:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775735.099640:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775735.099644:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775735.099646:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775735.099648:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775735.099649:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775735.099650:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775735.099653:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775735.099655:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775735.099658:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775735.099663:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775735.099666:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775735.099669:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775735.099672:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775735.099674:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775735.099678:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775735.099683:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775735.099688:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775735.099691:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775735.099694:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775735.099697:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775735.099699:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775735.099705:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775735.099707:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775735.099710:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775775735.099715:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775735.099718:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775735.099721:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775735.099723:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775735.099725:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775735.099727:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775735.099729:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000100:00000001:3.0:1775775735.099731:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775735.099735:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710361856/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775735.099740:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775735.099742:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775735.099747:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710361856/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775735.099757:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775735.099760:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775735.099766:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775775735.099781:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775775735.099786:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775775735.099788:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000100:00000040:3.0:1775775735.099792:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775735.099800:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12627) 00000800:00000200:1.1:1775775735.100225:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775735.100228:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775775735.100529:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775735.100532:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00010000:00000001:3.0:1775775735.227245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775735.227251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775735.227258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775735.227259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775735.227263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775735.227264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775735.227321:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775735.227325:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775735.227328:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775736.251237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775736.251242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775736.251246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775736.251247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775736.251250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775736.251251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775736.251317:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775736.251322:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775736.251325:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775737.275236:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775737.275242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775737.275249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775737.275250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775737.275254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775737.275255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775775737.275358:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775775737.275363:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775775737.275367:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775738.299260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775738.299267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775738.299272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775738.299276:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775738.299278:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775738.299281:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775738.299283:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775738.299285:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775738.299286:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775738.299288:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775738.299291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775738.299293:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775738.299294:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775738.299295:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775738.299296:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775738.299297:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775738.299298:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775738.299300:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775739.003224:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307293958, at 4307294016 00000100:00000010:0.0:1775775739.003236:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e29c0. 00010000:00000001:3.0:1775775739.323241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775739.323246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775739.323250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775739.323251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775739.323254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775739.323255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775739.323273:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775739.323274:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775739.323277:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775740.219247:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775740.219257:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775740.219268:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775740.219272:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775740.219276:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775740.219284:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12622 00000400:00000200:0.0:1775775740.219294:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775740.219300:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775740.219304:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775740.219307:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775740.219310:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775740.219313:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775740.219315:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775740.219319:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775740.219321:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775740.219324:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775740.219327:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775740.219330:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775740.219333:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775740.219336:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775740.219338:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775740.219341:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775740.219342:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775740.219346:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775740.219349:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775740.219352:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775740.219355:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775740.219359:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775740.219361:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775740.219363:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775740.219364:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775740.219371:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060380. 00000020:00000040:0.0:1775775740.219374:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775740.219377:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775740.219378:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775740.219380:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775740.219383:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775740.219386:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775740.219393:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7000. 00000100:00000001:0.0:1775775740.219398:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775740.219406:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060380 x1862022710361984/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775740.219414:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775740.219418:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775740.219421:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000614ace82] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060380 x1862022710361984/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775740.219445:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775740.219449:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775740.219453:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775740.219454:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775740.219457:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775740.219460:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775740.219465:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12622 00000400:00000200:0.0:1775775740.219474:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775740.219482:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775740.219488:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775740.219496:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775740.219500:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775740.219504:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775740.219507:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775740.219513:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775740.219515:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775740.219518:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775740.219522:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775740.219527:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775740.219530:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775740.219538:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775740.219540:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775740.219543:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775740.219545:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775740.219549:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775740.219552:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775740.219554:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775740.219555:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775740.219563:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775740.219569:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775740.219572:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775740.219578:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775740.219582:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775740.219585:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775740.219600:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063800. 00000020:00000040:0.0:1775775740.219606:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775740.219611:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775740.219615:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775740.219619:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775740.219624:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775740.219627:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775740.219636:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7c00. 00000100:00000001:0.0:1775775740.219643:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775740.219654:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063800 x1862022710362112/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775740.219700:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775740.219705:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775740.219712:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000015d81d25] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063800 x1862022710362112/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775740.219733:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775740.219737:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775740.219743:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775740.219744:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775740.219745:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.1:1775775740.219749:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775775740.219750:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775740.219754:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12622 00000800:00000200:1.1:1775775740.219756:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000400:00000200:0.0:1775775740.219761:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775740.219767:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775740.219772:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775740.219778:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775740.219782:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775740.219784:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775740.219786:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775740.219791:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775740.219793:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775740.219796:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775740.219798:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775740.219808:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775740.219812:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775740.219818:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775740.219823:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775740.219828:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775740.219833:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775740.219839:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775740.219846:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775740.219849:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775740.219853:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775740.219858:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775740.219863:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775740.219867:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775740.219870:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775740.219874:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775740.219879:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775740.219884:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062680. 00000020:00000040:0.0:1775775740.219888:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775740.219892:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775740.219895:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775740.219899:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775740.219903:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775740.219905:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775740.219913:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4800. 00000100:00000001:0.0:1775775740.219925:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775740.219934:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062680 x1862022710362240/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775740.219951:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775740.219956:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775740.219964:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000082c7985e] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062680 x1862022710362240/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775740.219993:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775740.219998:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775740.220006:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775740.220013:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775740.220020:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775740.220027:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12632) 00000800:00000200:1.1:1775775740.220324:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775740.220326:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775775740.220608:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775740.220610:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00010000:00000001:3.0:1775775740.347210:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775740.347214:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775740.347219:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775740.347220:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775740.347223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775740.347224:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775740.347279:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775740.347284:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775740.347288:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775741.371242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775741.371247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775741.371251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775741.371251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775741.371255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775741.371255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775741.371318:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775741.371323:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775741.371327:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775742.395268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775742.395273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775742.395278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775742.395279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775742.395282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775742.395283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775742.395344:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775742.395348:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775742.395351:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775743.419238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775743.419246:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775743.419249:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775743.419252:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775743.419258:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775743.419260:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775743.419262:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775743.419265:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775743.419269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775743.419271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775743.419274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775743.419276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775744.123237:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307299078, at 4307299136 00000100:00000010:0.0:1775775744.123251:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e22a0. 00010000:00000001:3.0:1775775744.443247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775744.443254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775744.443259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775744.443260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775744.443264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775744.443264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775744.443284:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775744.443287:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775744.443290:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775745.339224:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775745.339232:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775745.339236:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775745.339240:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775745.339243:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775775745.339243:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775745.339248:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1775775745.339250:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12627 00000100:00000001:2.0:1775775745.339251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775745.339257:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775745.339259:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775745.339262:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775775745.339264:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:2.0:1775775745.339265:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775745.339268:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775745.339271:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775745.339275:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775745.339279:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775745.339284:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775745.339288:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775745.339291:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775745.339293:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775745.339298:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775745.339299:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775745.339302:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775745.339305:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775745.339308:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775745.339310:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775745.339313:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775745.339314:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775745.339315:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775745.339317:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775745.339319:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775745.339321:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775745.339322:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775745.339325:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775745.339327:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775745.339329:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775745.339330:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775745.339333:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775745.339340:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062d80. 00000020:00000040:0.0:1775775745.339342:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775745.339346:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775745.339347:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775745.339349:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775745.339351:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775745.339355:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775745.339363:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5c00. 00000100:00000001:0.0:1775775745.339369:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775745.339376:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062d80 x1862022710362368/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775745.339384:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775745.339388:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775745.339391:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009965f80b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062d80 x1862022710362368/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775745.339407:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775745.339409:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775745.339412:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775745.339413:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775745.339414:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775745.339416:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775745.339418:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12627 00000400:00000200:0.0:1775775745.339421:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775745.339423:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775745.339425:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775745.339428:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775745.339430:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775745.339431:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775745.339433:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775745.339434:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775745.339436:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775745.339437:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775745.339439:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775745.339443:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775745.339445:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775745.339449:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775745.339452:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775745.339453:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775745.339456:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775745.339458:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775745.339462:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775745.339465:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775745.339468:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775745.339473:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775745.339477:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775745.339481:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775745.339485:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775745.339487:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775745.339491:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775745.339496:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062a00. 00000020:00000040:0.0:1775775745.339499:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775745.339503:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775745.339506:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775745.339507:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775745.339509:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775745.339511:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775745.339515:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4800. 00000100:00000001:0.0:1775775745.339520:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775745.339525:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062a00 x1862022710362496/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775745.339534:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775745.339537:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775745.339541:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009753f9f6] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062a00 x1862022710362496/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775745.339548:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775745.339551:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775745.339553:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775745.339554:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775745.339556:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775745.339559:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775745.339563:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12627 00000400:00000200:0.0:1775775745.339569:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775745.339572:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775745.339576:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775745.339581:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775745.339585:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775745.339588:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775745.339605:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775745.339607:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775745.339609:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775745.339611:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775745.339614:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775745.339618:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775745.339620:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775745.339626:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775745.339628:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775745.339629:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775745.339631:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775745.339634:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775745.339638:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775745.339640:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775745.339644:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775745.339648:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775745.339651:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775745.339653:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775745.339656:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775745.339659:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775745.339660:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775745.339665:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061500. 00000020:00000040:0.0:1775775745.339669:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775745.339673:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775745.339676:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775745.339678:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775745.339679:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775745.339680:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775745.339683:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7c00. 00000100:00000001:0.0:1775775745.339686:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775745.339692:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061500 x1862022710362624/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775745.339708:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775745.339710:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775745.339717:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb8d992] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061500 x1862022710362624/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775745.339726:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775745.339728:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775745.339734:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775745.339739:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775745.339746:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775775745.339750:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:0.0:1775775745.339753:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12638) 00000800:00000200:1.1:1775775745.339754:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775775745.340208:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775745.340210:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775775745.340496:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775745.340497:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00010000:00000001:3.0:1775775745.467199:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775745.467203:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775745.467207:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775745.467208:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775745.467211:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775745.467212:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775745.467279:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775745.467284:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775745.467287:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775746.363237:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775746.363243:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775746.363244:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775746.363249:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775746.363251:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775746.363254:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775746.363256:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775746.363258:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775746.363259:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775746.491255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775746.491262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775746.491267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775746.491268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775746.491271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775746.491272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775746.491333:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775746.491340:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775746.491345:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775747.387231:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775747.387237:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775747.387239:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775747.387243:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775747.387247:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775747.387250:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775747.387253:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775747.387255:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775747.387257:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775747.515216:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775747.515221:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775747.515225:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775747.515226:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775747.515228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775747.515229:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775747.515288:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775747.515296:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775747.515301:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775748.411259:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775748.411265:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775748.411266:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775748.411271:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775748.411274:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775748.411277:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775748.411279:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775748.411280:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775748.411282:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775748.539222:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775748.539227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775748.539231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775748.539235:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775748.539236:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775748.539238:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775748.539240:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775748.539242:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775748.539243:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775748.539245:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775748.539247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775748.539248:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775748.539250:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775748.539251:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775748.539252:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775748.539252:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775748.539253:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775748.539255:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775749.243240:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307304198, at 4307304256 00000100:00000010:0.0:1775775749.243263:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2360. 00000100:00000001:2.0:1775775749.435194:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775749.435200:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775749.435202:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775749.435206:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775749.435209:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775749.435211:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775749.435213:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775749.435215:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775749.435217:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775749.563269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775749.563274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775749.563278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775749.563279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775749.563282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775749.563283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775749.563304:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775749.563305:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775749.563308:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775750.459336:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775750.459345:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775750.459348:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775750.459352:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775775750.459355:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:3.0:1775775750.459361:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775750.459363:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775750.459365:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775775750.459368:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775775750.459370:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775750.459372:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775750.459375:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775775750.459376:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775750.459381:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775750.459384:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12633 00000400:00000200:3.0:1775775750.459395:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775750.459401:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775750.459404:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775750.459408:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775750.459412:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775750.459415:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775750.459417:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775750.459420:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775750.459422:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775750.459424:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775750.459426:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775750.459428:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775750.459430:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775750.459432:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775750.459434:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775750.459435:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775750.459436:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775750.459439:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775750.459442:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775750.459444:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775750.459447:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775750.459449:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775750.459452:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775750.459454:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775750.459456:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775750.459464:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73b80. 00000020:00000040:3.0:1775775750.459479:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775750.459483:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775750.459485:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775750.459487:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775750.459490:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775750.459494:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775750.459549:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000100:00000001:3.0:1775775750.459556:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775750.459566:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73b80 x1862022710362752/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775750.459575:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775750.459581:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775750.459585:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c52008f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73b80 x1862022710362752/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775750.459608:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775750.459611:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775750.459614:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775750.459615:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775750.459617:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775750.459619:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775750.459621:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12633 00000400:00000200:3.0:1775775750.459625:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775750.459628:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775750.459630:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775750.459632:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775750.459634:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775750.459636:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775750.459638:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775750.459640:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775750.459641:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775750.459642:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775750.459644:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775750.459646:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775750.459648:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775750.459651:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775750.459653:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775750.459655:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775750.459657:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775750.459660:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775750.459662:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775750.459667:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775750.459668:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775750.459675:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775750.459679:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775750.459682:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775750.459688:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775750.459691:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775750.459697:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775750.459705:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775775750.459709:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775750.459714:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775750.459734:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775750.459737:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775750.459741:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775750.459746:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775750.459751:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000100:00000001:3.0:1775775750.459758:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775750.459766:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710362880/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775750.459779:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775750.459784:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775750.459791:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710362880/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775750.459813:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775750.459816:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775750.459822:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775750.459825:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775750.459829:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775750.459833:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775750.459838:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12633 00000400:00000200:3.0:1775775750.459849:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775750.459856:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775750.459860:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775750.459867:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775750.459871:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775750.459875:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775750.459878:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775750.459881:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775750.459883:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775750.459886:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775750.459888:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775750.459890:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775750.459894:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775750.459900:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775750.459904:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775750.459909:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775750.459913:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775750.459916:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775750.459923:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775750.459928:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775750.459934:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775750.459939:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775750.459945:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775750.459950:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775750.459954:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775750.459959:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775750.459962:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775750.459966:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775775750.459971:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775750.459976:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775750.459978:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775750.459981:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775750.459984:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775750.460003:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775750.460010:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000100:00000001:3.0:1775775750.460016:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775750.460030:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710363008/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1775775750.460036:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775750.460043:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000100:00000001:3.0:1775775750.460059:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775750.460066:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775750.460075:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710363008/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775750.460115:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775750.460123:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775750.460137:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775750.460208:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775750.460212:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775750.460217:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12643) 00000800:00000200:1.1:1775775750.460695:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775750.460697:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775775750.460998:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775750.461000:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00010000:00000001:3.0:1775775750.587219:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775750.587224:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775750.587228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775750.587230:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775750.587232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775750.587233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775750.587296:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775750.587301:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775750.587305:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775751.483204:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775751.483211:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775751.483213:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775751.483218:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775751.483222:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775751.483226:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775751.483229:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775751.483231:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775751.483234:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775751.611242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775751.611248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775751.611252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775751.611253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775751.611256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775751.611257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775751.611327:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775751.611330:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775751.611333:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775775752.507303:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775775752.507311:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775752.507313:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775752.507317:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775752.507320:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775752.507324:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775752.507326:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775752.507329:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775752.507331:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775752.635215:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775752.635220:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775752.635224:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775752.635225:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775752.635228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775752.635229:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775752.635281:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775752.635285:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775752.635288:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775753.659254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775753.659262:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775753.659264:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775753.659268:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775753.659273:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775753.659275:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775753.659278:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775753.659280:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775753.659284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775753.659285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775753.659289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775753.659290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775754.363224:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307309319, at 4307309376 00000100:00000010:0.0:1775775754.363242:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2420. 00010000:00000001:3.0:1775775754.683267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775754.683275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775754.683282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775754.683283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775754.683287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775754.683288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775754.683314:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775754.683317:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775754.683320:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775755.579249:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775755.579256:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775755.579276:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775755.579279:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775755.579283:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775755.579289:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12638 00000400:00000200:0.0:1775775755.579300:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775755.579306:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775755.579308:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775755.579311:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775755.579314:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775755.579317:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775755.579319:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775755.579322:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775755.579323:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775755.579343:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775755.579345:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775755.579347:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775755.579349:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775755.579351:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775755.579353:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775755.579355:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775755.579356:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775755.579358:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775755.579361:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775755.579362:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775755.579365:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775755.579368:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775755.579371:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775755.579373:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775755.579375:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775755.579382:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063b80. 00000020:00000040:0.0:1775775755.579385:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775755.579388:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775755.579390:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775755.579392:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775755.579395:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775755.579398:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775755.579405:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5800. 00000100:00000001:0.0:1775775755.579410:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775755.579434:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063b80 x1862022710363136/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775755.579443:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775755.579447:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775755.579472:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ee9007b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063b80 x1862022710363136/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775755.579493:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775755.579495:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775755.579498:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775755.579499:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775755.579500:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775755.579503:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775755.579504:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12638 00000400:00000200:0.0:1775775755.579508:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775755.579510:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775755.579513:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775755.579516:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775755.579518:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775755.579519:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775755.579522:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775755.579524:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775755.579527:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775755.579530:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775755.579532:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775755.579534:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775755.579537:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775755.579542:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775755.579546:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775755.579550:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775755.579555:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775755.579559:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775755.579565:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775755.579571:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775755.579575:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775755.579581:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775755.579585:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775755.579587:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775755.579593:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775755.579597:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775755.579603:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775755.579607:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061f80. 00000020:00000040:0.0:1775775755.579613:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775755.579619:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775755.579621:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775755.579623:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775755.579626:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775755.579631:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775755.579656:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6c00. 00000100:00000001:0.0:1775775755.579663:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775755.579674:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061f80 x1862022710363264/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775755.579725:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775755.579730:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775755.579739:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000029d3794] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061f80 x1862022710363264/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775755.579754:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775755.579758:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775755.579761:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775755.579763:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775755.579766:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775755.579769:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775755.579774:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12638 00000400:00000200:0.0:1775775755.579813:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775755.579819:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775755.579823:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775755.579827:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775755.579831:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775755.579836:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775755.579839:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775755.579844:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775755.579846:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775755.579849:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:1.1:1775775755.579850:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000040:0.0:1775775755.579852:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775755.579856:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:1.1:1775775755.579856:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000020:00000001:0.0:1775775755.579862:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775755.579869:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775755.579873:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775755.579876:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775755.579879:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775755.579885:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775755.579891:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775755.579894:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775755.579896:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775755.579901:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775755.579904:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775755.579905:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775755.579909:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775755.579914:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775755.579918:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775755.579925:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060700. 00000020:00000040:0.0:1775775755.579930:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775755.579933:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775755.579936:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775755.579940:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775755.579943:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775755.579945:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775755.579951:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6400. 00000100:00000001:0.0:1775775755.579956:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775755.579965:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060700 x1862022710363392/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775755.579981:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775755.579984:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775755.579990:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000225c8695] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060700 x1862022710363392/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775755.579999:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775755.580005:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775755.580012:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775755.580016:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775755.580023:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775755.580033:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12648) 00000800:00000200:1.1:1775775755.580299:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775755.580301:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775775755.580615:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775755.580618:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00010000:00000001:3.0:1775775755.707285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775755.707293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775755.707298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775755.707299:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775755.707302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775755.707303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775755.707433:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775755.707439:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775755.707444:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775756.731300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775756.731309:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775756.731316:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775756.731317:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775756.731322:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775756.731323:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775756.731440:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775756.731447:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775756.731452:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775757.755289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775757.755295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775757.755302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775757.755303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775757.755308:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775757.755309:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775757.755385:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775757.755388:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775757.755391:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775758.779247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775758.779253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775758.779257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775758.779259:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775758.779261:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775758.779264:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775758.779266:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775758.779267:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775758.779269:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775758.779271:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775758.779273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775758.779274:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775758.779275:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775758.779276:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775758.779277:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775758.779278:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775758.779279:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775758.779281:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775759.483234:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307314438, at 4307314496 00000800:00000200:2.0:1775775759.483243:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000100:00000010:0.0:1775775759.483247:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2600. 00000800:00000200:2.0:1775775759.483253:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775775759.483258:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1775775759.483424:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775775759.483441:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775775759.483445:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775775759.483448:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1775775759.803272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775759.803279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775759.803287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775759.803289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775759.803294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775759.803296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775759.803316:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775759.803318:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775759.803320:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775760.699238:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775760.699255:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775760.699262:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775760.699265:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775760.699271:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775760.699278:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12643 00000400:00000200:0.0:1775775760.699289:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775760.699297:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775760.699301:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775760.699303:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775760.699306:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775760.699308:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775760.699310:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775760.699313:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775760.699315:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775760.699317:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775760.699319:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775760.699334:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775760.699336:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775760.699339:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775760.699340:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775760.699341:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775760.699343:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775760.699345:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775760.699348:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775760.699350:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775760.699353:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775760.699355:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775760.699357:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775760.699359:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775760.699361:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775760.699368:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061880. 00000020:00000040:0.0:1775775760.699371:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775760.699374:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775760.699375:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775760.699378:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775760.699380:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775760.699382:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775760.699389:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6000. 00000100:00000001:0.0:1775775760.699394:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775760.699401:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061880 x1862022710363520/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775760.699408:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775760.699412:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775760.699415:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005afd1f27] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061880 x1862022710363520/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775760.699432:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775760.699434:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775760.699437:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775760.699438:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775760.699439:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775760.699441:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775760.699443:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12643 00000400:00000200:0.0:1775775760.699446:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775760.699448:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775760.699451:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775760.699455:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775760.699458:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775760.699460:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775760.699463:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775760.699467:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775760.699470:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775760.699474:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775760.699476:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775760.699482:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775760.699488:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775760.699498:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775760.699505:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775760.699510:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775760.699513:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775760.699517:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775760.699521:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775760.699525:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775760.699528:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775760.699534:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775760.699538:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775760.699542:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775760.699547:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775760.699551:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775760.699552:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775760.699557:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061180. 00000020:00000040:0.0:1775775760.699560:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775760.699564:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775760.699566:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775760.699567:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775760.699569:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775760.699572:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775760.699578:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4400. 00000100:00000001:0.0:1775775760.699584:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775760.699590:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061180 x1862022710363648/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775760.699598:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775760.699600:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775760.699604:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e23859] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061180 x1862022710363648/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775760.699615:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775760.699626:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775760.699633:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775760.699636:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775760.699639:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775760.699646:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775760.699652:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12643 00000400:00000200:0.0:1775775760.699659:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775760.699665:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775760.699667:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775760.699671:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775760.699673:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775760.699678:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775760.699682:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775760.699684:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775760.699685:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775760.699686:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775760.699687:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775760.699691:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775760.699695:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775760.699698:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775760.699702:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775760.699708:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775760.699709:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775760.699712:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775760.699716:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775760.699719:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775760.699723:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775760.699725:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775760.699728:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775760.699731:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775760.699734:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775760.699738:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775760.699739:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775760.699744:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061c00. 00000020:00000040:0.0:1775775760.699748:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775760.699753:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775760.699755:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775760.699757:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775760.699760:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775760.699763:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775760.699765:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000001:0.0:1775775760.699770:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775760.699777:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061c00 x1862022710363776/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775760.699786:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775760.699789:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775760.699795:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076a44879] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061c00 x1862022710363776/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775760.699803:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775760.699805:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775760.699812:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775760.699816:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775760.699823:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775775760.699827:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775760.699831:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00000040:0.0:1775775760.699832:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12653) 00000800:00000200:1.1:1775775760.700129:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775760.700131:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775775760.700382:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775760.700383:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00010000:00000001:3.0:1775775760.827264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775760.827271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775760.827275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775760.827276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775760.827279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775760.827280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775760.827430:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775760.827434:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775760.827437:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775761.851290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775761.851300:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775761.851333:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775761.851335:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775761.851339:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775761.851341:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775761.851438:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775761.851445:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775761.851450:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775762.875237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775762.875243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775762.875249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775762.875250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775762.875253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775762.875254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775762.875344:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775762.875349:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775762.875353:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775763.771387:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775763.771394:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775763.771396:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775763.771401:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775763.771404:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775763.771406:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775763.771408:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775763.771411:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775763.771413:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775763.899237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775763.899245:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775763.899247:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775763.899251:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775763.899255:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775763.899258:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775763.899259:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775763.899262:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775763.899300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775763.899302:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775763.899307:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775763.899308:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775764.603232:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307319558, at 4307319616 00000100:00000010:0.0:1775775764.603247:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2720. 00000100:00000001:3.0:1775775764.795265:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775764.795273:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775764.795275:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775764.795280:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775764.795283:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775764.795286:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775764.795289:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775764.795291:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775764.795293:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775764.923212:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775764.923217:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775764.923223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775764.923224:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775764.923228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775764.923230:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775764.923269:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775764.923272:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775764.923275:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775765.819231:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775765.819249:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775765.819250:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775765.819255:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775765.819259:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775765.819262:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775765.819264:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775765.819267:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775765.819269:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775765.819470:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775765.819474:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775765.819476:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775765.819482:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234919f80 x1862022710363904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.819491:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775765.819493:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775765.819496:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775765.819499:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775765.819501:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775765.819505:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234919f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710363904:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775765.819510:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775765.819513:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775765.819516:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775765.819518:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775765.819523:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775775765.819526:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775765.819531:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775765.819535:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775765.819543:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775765.819549:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710363904, portal 25 00000100:00000001:0.0:1775775765.819552:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775765.819553:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684366720 : -123823025184896 : ffff8f6234919f80) 00000100:00000040:0.0:1775775765.819558:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234919f80 x1862022710363904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775790 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.819566:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775765.819569:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775765.819572:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775765.819576:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710363904, offset 0 00000400:00000200:0.0:1775775765.819581:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775765.819589:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775765.819594:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775765.819599:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775765.819603:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775765.819607:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775765.819610:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775765.819615:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453395:453395:256:4294967295] 192.168.201.115@tcp LPNI seq info [453320:453320:8:4294967295] 00000400:00000200:0.0:1775775765.819620:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775765.819630:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775765.819636:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775765.819642:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775765.819648:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775765.819652:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775775765.819658:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775765.819662:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775775765.819667:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775765.819670:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775765.819676:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775765.819681:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775765.819686:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775765.819688:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775765.819692:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775765.819695:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234919f80 x1862022710363904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775790 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.819704:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775765.819707:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234919f80 x1862022710363904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775790 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.819714:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775765.819716:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775765.819723:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775765.819727:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775765.819729:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307324678 00000100:00000010:0.0:1775775765.819734:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775765.819741:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775765.819747:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307324678 00000100:00000010:0.0:1775775765.819749:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3840. 00000100:00000001:0.0:1775775765.819751:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775765.819753:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775765.819755:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775765.819757:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775765.819760:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775765/real 1775775765] req@ffff8f6234919f80 x1862022710363904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775765.819768:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775765.819770:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775765.819772:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775765.819774:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775765.819775:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775765.819777:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775765.819779:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234919f80 x1862022710363904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775765.819785:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234919f80 x1862022710363904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.819789:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775765.819790:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775765.819791:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775765.819793:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775765.819795:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234919f80 x1862022710363904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.819799:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775765.819802:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234919f80 x1862022710363904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.819808:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775765.819810:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775765.819811:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775765.819812:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775765.819813:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775765.819815:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775765.819816:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775765.819819:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775765.819821:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775765.819823:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234919f80 x1862022710363904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775765.819828:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234919f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710363904:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775765.819831:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775765.819832:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775765.819834:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234919f80 x1862022710363904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.819838:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775765.819841:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775765.819842:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775775765.819844:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775765.819845:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775765.819846:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775765.819848:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775765.819850:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:0.0:1775775765.819852:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775765.819853:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775765.819854:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234919f80. 00000100:00000001:0.0:1775775765.819857:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775765.819857:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775765.819859:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775765.819865:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775765.819866:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775765.819867:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775765.819868:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775765.819869:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775765.819871:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491a680 x1862022710364032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.819876:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775765.819877:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775765.819879:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775765.819880:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775765.819881:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775765.819883:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710364032:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775765.819885:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775765.819886:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775765.819887:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775765.819888:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775765.819890:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775775765.819891:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775765.819894:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775765.819896:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775765.819898:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775765.819901:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710364032, portal 10 00000100:00000001:0.0:1775775765.819902:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775765.819903:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684368512 : -123823025183104 : ffff8f623491a680) 00000100:00000040:0.0:1775775765.819907:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491a680 x1862022710364032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775790 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.819911:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775765.819913:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775765.819914:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775765.819916:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710364032, offset 0 00000400:00000200:0.0:1775775765.819918:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775765.819921:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775765.819924:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775765.819926:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775765.819928:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775765.819930:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775765.819931:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775765.819933:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453396:453396:256:4294967295] 192.168.201.115@tcp LPNI seq info [453321:453321:8:4294967295] 00000400:00000200:0.0:1775775765.819936:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775765.819940:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775765.819944:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775765.819946:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775765.819948:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775765.819950:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775775765.819953:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775765.819955:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775775765.819957:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775765.819959:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775765.819961:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775765.819963:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775765.819965:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775765.819966:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775765.819968:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775765.819970:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491a680 x1862022710364032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775790 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.819974:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775765.819976:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491a680 x1862022710364032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775790 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.819980:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775765.819981:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775765.819983:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775765.819985:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775765.819987:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775765.819989:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307320843 00000100:00000001:0.0:1775775765.819991:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307324678 00000100:00000001:0.0:1775775765.819992:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307320843 00000100:00000001:0.0:1775775765.819994:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307324678 00000100:00000001:0.0:1775775765.819995:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775765.819996:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775765.819998:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775765.820000:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775765/real 1775775765] req@ffff8f623491a680 x1862022710364032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775765.820004:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775765.820005:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775765.820006:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775765.820008:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775765.820009:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775765.820010:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775765.820012:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491a680 x1862022710364032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775765.820017:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491a680 x1862022710364032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.820021:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775765.820021:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775765.820022:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775765.820023:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775765.820025:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491a680 x1862022710364032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.820029:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775765.820031:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491a680 x1862022710364032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.820035:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775765.820037:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775765.820038:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775765.820039:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775765.820041:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775765.820046:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775765.820049:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775765.820052:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775765.820054:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775765.820059:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491a680 x1862022710364032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775765.820076:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710364032:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775765.820081:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775765.820085:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491a300 x1862022710364160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.820092:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775765.820094:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775765.820096:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775765.820098:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775765.820100:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775765.820102:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710364160:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775765.820106:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775765.820108:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775765.820109:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775765.820110:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775765.820113:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775765.820114:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775765.820117:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775765.820118:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775765.820120:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775765.820123:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710364160, portal 10 00000100:00000001:0.0:1775775765.820124:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775765.820126:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684367616 : -123823025184000 : ffff8f623491a300) 00000100:00000040:0.0:1775775765.820130:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491a300 x1862022710364160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775790 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.820152:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775765.820154:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775765.820156:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775765.820158:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710364160, offset 0 00000400:00000200:0.0:1775775765.820160:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775765.820163:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775765.820166:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775765.820168:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775765.820169:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775765.820171:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775765.820173:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775765.820175:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453397:453397:256:4294967295] 192.168.201.115@tcp LPNI seq info [453322:453322:8:4294967295] 00000400:00000200:0.0:1775775765.820178:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775765.820182:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775765.820185:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775765.820187:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775765.820189:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775765.820191:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775775765.820194:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775765.820197:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775775765.820199:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775765.820201:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775765.820203:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775765.820205:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775765.820207:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775765.820208:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775765.820210:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775765.820212:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491a300 x1862022710364160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775790 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.820221:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775765.820223:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491a300 x1862022710364160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775790 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.820228:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775765.820229:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775765.820230:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775765.820248:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775765.820252:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775765.820253:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307320844 00000100:00000001:0.0:1775775765.820255:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307324678 00000100:00000001:0.0:1775775765.820256:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307320844 00000100:00000001:0.0:1775775765.820258:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307324678 00000100:00000001:0.0:1775775765.820259:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775765.820260:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775765.820262:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775765.820264:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775765/real 1775775765] req@ffff8f623491a300 x1862022710364160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775765.820270:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775765.820270:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775765.820272:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888318. 00000400:00000200:0.0:1775775765.820273:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc06e8 00000400:00000010:0.0:1775775765.820274:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc06e8. 00000100:00000001:0.0:1775775765.820276:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775765.820278:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491a300 x1862022710364160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775765.820283:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491a300 x1862022710364160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.820287:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775765.820289:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775765.820290:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775765.820291:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775765.820293:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491a300 x1862022710364160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.820297:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775765.820299:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491a300 x1862022710364160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.820303:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775765.820305:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775765.820306:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775765.820306:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775765.820307:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775765.820309:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775765.820310:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775765.820312:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775765.820313:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775765.820315:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491a300 x1862022710364160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775765.820321:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710364160:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775765.820323:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775765.820324:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775765.820326:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491a680 x1862022710364032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.820330:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775765.820331:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775765.820332:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775775765.820334:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775765.820334:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775765.820336:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775765.820337:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775765.820338:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4c000. 02000000:00000001:0.0:1775775765.820340:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775765.820341:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775765.820342:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491a680. 00000100:00000001:0.0:1775775765.820343:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775765.820344:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775765.820346:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775765.820347:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491a300 x1862022710364160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775790 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775765.820352:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775765.820352:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775765.820353:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775765.820354:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775765.820355:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775765.820356:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775765.820357:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775765.820358:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:0.0:1775775765.820359:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775765.820359:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775765.820360:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491a300. 00000100:00000001:0.0:1775775765.820361:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775765.820362:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775765.820363:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775765.820365:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775765.820366:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775765.820367:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775765.820368:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775765.947362:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775765.947369:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775765.947376:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775765.947377:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775765.947382:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775765.947384:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775765.947451:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775765.947454:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775765.947458:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775766.843294:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775766.843304:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775766.843307:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775766.843315:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775766.843337:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775766.843342:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775766.843346:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775766.843350:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775766.843355:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775766.971301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775766.971306:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775766.971310:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775766.971312:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775766.971315:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775766.971316:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775766.971384:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775766.971389:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775766.971392:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775767.867257:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775767.867263:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775767.867265:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775767.867269:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775767.867272:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775767.867275:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775767.867278:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775767.867280:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775767.867282:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775767.995259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775767.995265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775767.995272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775767.995274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775767.995278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775767.995279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775767.995325:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775767.995328:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775767.995331:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775768.891243:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775768.891249:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775768.891250:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775768.891254:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775768.891258:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775768.891262:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775768.891264:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775768.891267:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775768.891270:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775769.019286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775769.019291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775769.019296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775769.019298:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775769.019300:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775769.019303:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775769.019305:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775769.019307:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775769.019308:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775769.019310:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775769.019313:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775769.019314:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775769.019315:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775769.019316:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775769.019318:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775769.019318:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775769.019320:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775769.019321:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775769.723255:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307324678, at 4307324736 00000100:00000010:0.0:1775775769.723273:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00000100:00000001:3.0:1775775769.915287:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775769.915297:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775769.915299:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775769.915308:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775769.915313:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775769.915319:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775769.915323:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775769.915326:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775769.915329:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775770.043301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775770.043307:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775770.043313:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775770.043326:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775770.043330:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775770.043331:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775770.043363:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775770.043367:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775770.043371:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775770.939268:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775770.939276:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775770.939284:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775770.939290:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775775770.939294:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775770.939299:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775775770.939301:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775770.939304:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775770.939306:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775775770.939307:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775770.939311:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775775770.939312:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775770.939314:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775770.939320:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775770.939328:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12653 00000400:00000200:0.0:1775775770.939341:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775770.939350:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775770.939354:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775770.939358:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775770.939361:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775770.939365:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775770.939368:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775770.939373:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775770.939375:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775770.939379:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775770.939382:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775770.939385:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775770.939387:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775770.939389:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775770.939391:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775770.939392:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775770.939394:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775770.939396:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775770.939399:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775770.939400:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775770.939403:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775770.939406:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775770.939409:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775770.939411:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775770.939412:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775770.939424:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060e00. 00000020:00000040:0.0:1775775770.939428:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775770.939439:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775770.939441:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775770.939445:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775770.939447:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775770.939452:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775770.939460:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6000. 00000100:00000001:0.0:1775775770.939467:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775770.939477:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060e00 x1862022710364288/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775770.939486:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775770.939491:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775770.939495:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004a3bddaa] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060e00 x1862022710364288/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775770.939524:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775770.939528:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775770.939532:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775770.939533:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775770.939535:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775770.939538:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775770.939542:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12653 00000400:00000200:0.0:1775775770.939546:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775770.939550:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775770.939554:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775770.939558:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775770.939561:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775770.939564:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775770.939565:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775770.939567:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775770.939569:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775770.939570:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775770.939571:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775770.939573:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775770.939574:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775770.939579:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775770.939582:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775770.939587:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775770.939590:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775770.939594:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775770.939596:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775770.939602:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775770.939604:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775770.939610:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775770.939616:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775770.939626:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775770.939663:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775770.939675:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775770.939682:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775770.939688:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063100. 00000020:00000040:0.0:1775775770.939693:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775770.939696:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775770.939697:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775770.939699:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775770.939700:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775770.939703:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775770.939712:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4400. 00000100:00000001:0.0:1775775770.939719:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775770.939749:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063100 x1862022710364416/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775770.939757:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775770.939762:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775770.939765:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033f6db7f] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063100 x1862022710364416/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775770.939790:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775770.939801:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775770.939819:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775770.939840:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775770.939847:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775770.939857:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775770.939863:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12653 00000400:00000200:0.0:1775775770.939870:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775770.939880:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775770.939882:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775770.939892:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775770.939895:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775770.939900:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775770.939909:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775770.939916:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775770.939917:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775770.939921:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775770.939923:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775770.939933:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775770.939942:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775770.939957:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775770.939963:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775770.939967:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775770.939970:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775770.939973:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775770.939981:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775770.939990:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775770.939998:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775770.940007:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775770.940018:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775770.940025:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775770.940028:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775770.940035:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775770.940046:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775770.940054:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061c00. 00000800:00000200:1.1:1775775770.940055:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000040:0.0:1775775770.940060:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000800:00000200:1.1:1775775770.940061:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 02000000:00000001:0.0:1775775770.940065:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775770.940070:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775770.940075:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775770.940080:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775770.940083:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775770.940093:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7800. 00000100:00000001:0.0:1775775770.940107:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775770.940121:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061c00 x1862022710364544/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775770.940195:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775770.940199:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775770.940208:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076a44879] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061c00 x1862022710364544/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775770.940254:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775770.940261:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775770.940270:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775770.940283:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775770.940300:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775770.940311:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12663) 00000800:00000200:1.1:1775775770.940642:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775770.940644:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775775770.940866:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775770.940867:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00010000:00000001:3.0:1775775771.067291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775771.067298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775771.067303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775771.067305:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775771.067308:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775771.067309:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775771.067375:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775771.067378:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775771.067381:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775771.963260:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775771.963266:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775771.963268:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775771.963274:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775771.963278:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775771.963283:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775771.963285:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775771.963288:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775771.963290:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775772.091236:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775772.091243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775772.091249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775772.091250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775772.091261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775772.091263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775772.091315:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775772.091319:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775772.091323:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775772.987230:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775772.987236:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775772.987238:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775772.987244:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775772.987248:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775772.987251:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775772.987254:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775772.987257:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775772.987260:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775773.115285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775773.115291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775773.115309:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775773.115310:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775773.115313:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775773.115314:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775773.115389:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775773.115395:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775773.115399:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775774.011270:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775774.011278:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775774.011281:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775774.011287:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775774.011292:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775774.011313:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775774.011317:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775774.011319:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775774.011322:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775774.139232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775774.139241:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775774.139244:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775774.139247:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775774.139253:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775774.139256:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775774.139259:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775774.139262:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775774.139267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775774.139268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775774.139273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775774.139274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775774.843254:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307329799, at 4307329856 00000100:00000010:0.0:1775775774.843274:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2a80. 00000100:00000001:3.0:1775775775.035229:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775775.035235:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775775.035236:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775775.035240:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775775.035243:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775775.035246:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775775.035249:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775775.035250:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775775.035252:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775775.163226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775775.163231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775775.163235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775775.163237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775775.163240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775775.163241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775775.163257:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775775.163259:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775775.163261:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775776.059253:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775776.059258:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775776.059260:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775776.059266:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775776.059269:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775776.059272:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775776.059275:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775775776.059276:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775775776.059277:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775776.059282:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775776.059285:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775776.059289:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775776.059295:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775776.059299:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775776.059307:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12658 00000400:00000200:0.0:1775775776.059318:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775776.059325:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775776.059329:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775776.059333:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775776.059336:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775776.059339:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775776.059341:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775776.059355:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775776.059357:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775776.059360:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775776.059363:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775776.059366:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775776.059368:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775776.059372:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775776.059374:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775776.059376:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775776.059378:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775776.059382:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775776.059385:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775776.059388:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775776.059392:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775776.059396:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775776.059399:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775776.059402:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775776.059404:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775776.059420:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061180. 00000020:00000040:0.0:1775775776.059425:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775776.059430:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775776.059432:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775776.059435:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775776.059438:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775776.059442:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775776.059451:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6c00. 00000100:00000001:0.0:1775775776.059457:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775776.059466:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061180 x1862022710364672/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775776.059476:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775776.059481:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775776.059486:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e23859] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061180 x1862022710364672/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775776.059506:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775776.059509:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775776.059512:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775776.059514:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775776.059516:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775776.059519:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775776.059527:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12658 00000400:00000200:0.0:1775775776.059536:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775776.059544:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775776.059547:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775776.059555:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775776.059560:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775776.059566:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775776.059572:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775776.059578:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775776.059581:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775776.059585:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775776.059589:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775776.059594:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775776.059599:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775776.059606:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775776.059609:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775776.059614:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775776.059618:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775776.059624:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775776.059630:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775776.059633:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775776.059637:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775776.059644:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775776.059649:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775776.059653:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775776.059659:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775776.059665:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775776.059670:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775776.059677:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061880. 00000020:00000040:0.0:1775775776.059684:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775776.059688:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775776.059689:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775776.059692:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775776.059695:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775776.059698:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775776.059704:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5800. 00000100:00000001:0.0:1775775776.059712:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775776.059720:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061880 x1862022710364800/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775776.059744:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775776.059748:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775776.059758:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005afd1f27] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061880 x1862022710364800/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775776.059775:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775776.059780:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775776.059786:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000200:1.1:1775775776.059786:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775775776.059789:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:1.1:1775775776.059790:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000100:00080000:0.0:1775775776.059792:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775776.059798:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775776.059804:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12658 00000400:00000200:0.0:1775775776.059814:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775776.059822:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775776.059827:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775776.059832:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775776.059836:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775776.059842:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775776.059848:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775776.059854:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775776.059857:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775776.059860:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775776.059863:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775776.059868:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775776.059874:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775776.059881:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775776.059885:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775776.059890:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775776.059896:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775776.059903:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775776.059908:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775776.059911:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775776.059917:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775776.059921:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775776.059928:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775776.059933:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775776.059938:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775776.059943:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775776.059948:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775776.059956:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060700. 00000020:00000040:0.0:1775775776.059960:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775776.059966:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775776.059970:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775776.059976:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775776.059980:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775776.059984:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775776.059991:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7000. 00000100:00000001:0.0:1775775776.059999:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775776.060008:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060700 x1862022710364928/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775776.060036:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775776.060041:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775776.060052:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000225c8695] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060700 x1862022710364928/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775776.060086:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775776.060090:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775776.060098:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775776.060105:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775776.060113:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775776.060123:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12668) 00000800:00000200:1.1:1775775776.060287:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775776.060289:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775775776.060518:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775776.060520:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00010000:00000001:3.0:1775775776.187247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775776.187252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775776.187255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775776.187256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775776.187259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775776.187260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775776.187342:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775776.187350:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775776.187356:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775777.083230:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775777.083236:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775777.083238:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775777.083243:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775777.083246:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775777.083250:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775777.083253:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775777.083255:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775777.083257:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775777.211234:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775777.211240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775777.211244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775777.211245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775777.211248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775777.211249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775777.211325:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775777.211331:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775777.211335:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775778.107257:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775778.107263:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775778.107265:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775778.107270:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775778.107273:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775778.107277:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775778.107280:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775778.107282:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775778.107284:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775778.235265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775778.235270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775778.235275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775778.235277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775778.235280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775778.235281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775778.235359:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775778.235364:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775778.235368:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775779.131236:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775779.131244:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775779.131247:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775779.131252:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775779.131256:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775779.131260:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775779.131263:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775779.131265:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775779.131267:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775779.259271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775779.259277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775779.259283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775779.259287:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775779.259289:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775779.259292:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775779.259295:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775779.259297:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775779.259299:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775779.259300:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775779.259303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775779.259304:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775779.259305:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775779.259306:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775779.259307:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775779.259308:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775779.259309:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775779.259310:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775779.963268:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307334918, at 4307334976 00000100:00000010:0.0:1775775779.963295:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2f60. 00000100:00000001:3.0:1775775780.155304:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775780.155311:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775780.155313:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775780.155325:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775780.155329:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775780.155333:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775780.155336:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775780.155339:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775780.155342:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775780.283271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775780.283278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775780.283284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775780.283285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775780.283288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775780.283289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775780.283316:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775780.283318:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775780.283321:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775781.179298:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775781.179308:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775781.179314:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775781.179321:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775781.179329:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775781.179334:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775781.179340:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775781.179349:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775781.179356:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775781.179544:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775781.179549:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775781.179553:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775781.179563:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234919c00 x1862022710365056/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775781.179584:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775781.179587:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775781.179592:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775781.179596:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775781.179599:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775781.179604:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234919c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710365056:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775781.179611:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775781.179616:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775781.179620:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775781.179626:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775781.179632:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775775781.179636:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775781.179647:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775781.179653:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888318. 00000400:00000010:0.0:1775775781.179665:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc06e8. 00000100:00000200:0.0:1775775781.179679:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710365056, portal 25 00000100:00000001:0.0:1775775781.179685:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775781.179690:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684365824 : -123823025185792 : ffff8f6234919c00) 00000100:00000040:0.0:1775775781.179702:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234919c00 x1862022710365056/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775781.179717:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775781.179723:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775781.179727:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775775781.179730:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710365056, offset 0 00000400:00000200:0.0:1775775781.179737:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775781.179749:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775781.179773:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775781.179783:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775781.179792:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775781.179811:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775781.179817:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775781.179823:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453404:453404:256:4294967295] 192.168.201.115@tcp LPNI seq info [453329:453329:8:4294967295] 00000400:00000200:0.0:1775775781.179833:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775781.179851:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775781.179866:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775781.179876:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775775781.179891:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775781.179900:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775775781.179920:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775781.179925:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775781.179928:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307340038 00000100:00000010:0.0:1775775781.179938:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775781.179947:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775781.179957:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307340038 00000100:00000010:0.0:1775775781.179961:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c36c0. 00000100:00000001:0.0:1775775781.179976:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775781.179981:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775781.179984:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775781.179988:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775781.179991:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775781.180000:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775781.180002:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1775775781.180005:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775781.180006:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775781.180008:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775781.180012:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491a300 x1862022710365184/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775781.180018:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775781.180020:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775781.180021:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775781.180023:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775781.180024:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775781.180026:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710365184:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775781.180031:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775781.180033:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775781.180034:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775781.180035:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775781.180038:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5c00. 02000000:00000001:0.0:1775775781.180040:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775781.180043:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775781.180047:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888738. 00000400:00000010:0.0:1775775781.180050:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0550. 00000100:00000200:0.0:1775775781.180053:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710365184, portal 10 00000100:00000001:0.0:1775775781.180054:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775781.180055:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684367616 : -123823025184000 : ffff8f623491a300) 00000100:00000040:0.0:1775775781.180060:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491a300 x1862022710365184/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775781.180066:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775781.180067:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775781.180070:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775775781.180073:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710365184, offset 0 00000400:00000200:0.0:1775775781.180079:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775781.180084:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775781.180088:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775781.180091:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775781.180094:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775781.180097:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775781.180100:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775781.180104:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453405:453405:255:4294967295] 192.168.201.115@tcp LPNI seq info [453330:453330:7:4294967295] 00000400:00000200:0.0:1775775781.180109:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775781.180117:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775781.180123:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775781.180129:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c600. 00000800:00000200:0.0:1775775781.180151:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775781.180155:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775775781.180157:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775781.180160:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775781.180163:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775781.180165:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307336204 00000100:00000001:0.0:1775775781.180168:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307340038 00000100:00000001:0.0:1775775781.180170:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307336204 00000100:00000001:0.0:1775775781.180173:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307340038 00000100:00000001:0.0:1775775781.180175:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775781.180177:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775781.180180:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775781.180184:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491a680 x1862022710365312/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775781.180192:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775781.180193:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775781.180196:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775781.180197:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775781.180200:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775781.180203:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710365312:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775781.180207:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775781.180209:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775781.180211:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775781.180212:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775781.180216:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7c00. 02000000:00000001:0.0:1775775781.180218:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775781.180222:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775781.180225:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888cb8. 00000400:00000010:0.0:1775775781.180229:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc03b8. 00000100:00000200:0.0:1775775781.180231:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710365312, portal 10 00000100:00000001:0.0:1775775781.180232:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775781.180233:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684368512 : -123823025183104 : ffff8f623491a680) 00000100:00000040:0.0:1775775781.180238:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491a680 x1862022710365312/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775781.180244:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775781.180245:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775781.180247:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775775781.180250:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710365312, offset 0 00000400:00000200:0.0:1775775781.180255:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775781.180259:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775781.180263:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775781.180266:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775781.180268:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775781.180271:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775781.180273:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775781.180276:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453406:453406:254:4294967295] 192.168.201.115@tcp LPNI seq info [453331:453331:6:4294967295] 00000400:00000200:0.0:1775775781.180281:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775781.180288:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775781.180293:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775781.180298:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775775781.180302:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775781.180306:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775775781.180309:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775781.180311:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775781.180315:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775781.180317:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307336204 00000100:00000001:0.0:1775775781.180320:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307340038 00000100:00000001:0.0:1775775781.180322:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307336204 00000100:00000001:0.0:1775775781.180325:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307340038 00000100:00000001:0.0:1775775781.180327:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775781.180329:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775781.180339:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775781.180341:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775781.180344:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775781.180356:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1775775781.180358:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775781.180359:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775781.180361:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775781.180363:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775781.307245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775781.307250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775781.307254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775781.307255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775781.307257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775781.307258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775781.307315:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775781.307318:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775781.307321:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775782.203291:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775782.203297:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775782.203298:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775782.203302:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775782.203305:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775782.203307:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775782.203309:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775782.203311:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775782.203313:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775782.331233:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775782.331237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775782.331241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775782.331242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775782.331246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775782.331247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775782.331306:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775782.331310:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775782.331313:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775783.227310:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775783.227315:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775783.227317:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775783.227321:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775783.227323:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775783.227326:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775783.227328:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775783.227330:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775783.227332:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775783.355230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775783.355236:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775783.355238:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775783.355240:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775783.355245:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775783.355246:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775783.355248:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775783.355249:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775783.355252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775783.355253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775783.355256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775783.355257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775783.355311:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775783.355314:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775783.355317:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775784.251401:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775784.251408:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775784.251410:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775784.251416:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775784.251419:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775784.251423:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000100:0.0:1775775784.251425:0:2402:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000100:00000001:3.0:1775775784.251425:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775784.251428:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775784.251431:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1775775784.251464:0:2402:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1775775784.251474:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775784.251478:0:2402:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:0.0:1775775784.251484:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775784.251488:0:2402:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:0.0:1775775784.251494:0:2402:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:0.0:1775775784.251498:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000200:0.0:1775775784.251502:0:2402:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775775784.251508:0:2402:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775775784.251531:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775775784.251533:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775775784.251539:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775784.251547:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6234919c00 x1862022710365056/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775784.251592:0:2402:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775784.251598:0:2402:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234919c00 x1862022710365056/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775806 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775784.251607:0:2402:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775784.251609:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775775784.251621:0:2402:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:0.0:1775775784.251624:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c600. 00000400:00000200:0.0:1775775784.251627:0:2402:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775775784.251633:0:2402:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775775784.251638:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775775784.251641:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775775784.251647:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775784.251653:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f623491a300 x1862022710365184/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775784.251672:0:2402:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775784.251679:0:2402:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491a300 x1862022710365184/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775806 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775784.251693:0:2402:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775784.251696:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775775784.251706:0:2402:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:0.0:1775775784.251711:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775775784.251715:0:2402:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775775784.251723:0:2402:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775775784.251732:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775775784.251736:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775775784.251742:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775784.251749:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f623491a680 x1862022710365312/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775784.251769:0:2402:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775784.251776:0:2402:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491a680 x1862022710365312/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775806 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775784.251791:0:2402:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775784.251795:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:3.0:1775775784.379232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775784.379238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775784.379244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775784.379246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775784.379249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775784.379250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775785.083229:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307340038, at 4307340096 00000100:00000010:0.0:1775775785.083243:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00000100:00000001:3.0:1775775785.275314:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775785.275319:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775785.275321:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775785.275324:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775785.275327:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775785.275330:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775785.275333:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775785.275334:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775785.275336:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775785.403234:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775785.403239:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775785.403243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775785.403244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775785.403246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775785.403247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775785.403266:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775785.403268:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775785.403270:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775786.299295:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775775786.299300:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775786.299305:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1775775786.299307:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775786.299308:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775786.299311:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1775775786.299313:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:0.0:1775775786.299315:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775786.299318:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775786.299320:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775775786.299322:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775786.299325:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775775786.299328:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12668 00000100:00000001:3.0:1775775786.299329:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775775786.299339:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775775786.299339:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775786.299347:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775786.299351:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775786.299355:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775786.299359:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775786.299362:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775786.299365:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775786.299369:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775786.299372:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775786.299375:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775786.299378:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775786.299382:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775786.299385:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775786.299388:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775786.299391:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775786.299393:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775786.299395:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775786.299399:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775786.299404:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775786.299407:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775786.299411:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775786.299414:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775786.299418:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775786.299421:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775786.299424:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775786.299432:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061f80. 00000020:00000040:0.0:1775775786.299437:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775786.299442:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775786.299444:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775786.299448:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775786.299452:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775786.299458:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775786.299469:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7000. 00000100:00000001:0.0:1775775786.299477:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775786.299490:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061f80 x1862022710365440/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775786.299502:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775786.299508:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775786.299515:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000029d3794] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061f80 x1862022710365440/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775786.299539:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775786.299543:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775786.299547:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775786.299548:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775786.299551:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775786.299553:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775786.299556:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12668 00000400:00000200:0.0:1775775786.299562:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775786.299565:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775786.299568:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775786.299573:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775786.299575:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775786.299578:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775786.299583:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775786.299587:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775786.299590:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775786.299593:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775786.299597:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775786.299601:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775786.299608:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775786.299616:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775786.299621:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775786.299626:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775786.299632:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775786.299637:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775786.299644:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775786.299649:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775786.299652:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775786.299659:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775786.299664:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775786.299667:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775786.299673:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775786.299679:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775786.299682:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775786.299687:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063b80. 00000020:00000040:0.0:1775775786.299692:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775786.299698:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775786.299700:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775786.299702:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775786.299705:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775786.299709:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775786.299716:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5800. 00000100:00000001:0.0:1775775786.299722:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775786.299729:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063b80 x1862022710365568/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775786.299742:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775786.299748:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775786.299756:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ee9007b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063b80 x1862022710365568/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775786.299786:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775786.299792:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775786.299798:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775786.299801:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775786.299803:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775786.299809:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775786.299814:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12668 00000400:00000200:0.0:1775775786.299822:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775786.299826:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775786.299831:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775786.299837:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775786.299840:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775786.299846:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775786.299851:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775786.299853:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775786.299856:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775786.299860:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775786.299864:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775786.299868:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775786.299873:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775786.299894:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775786.299898:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775786.299903:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775786.299908:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775786.299914:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775786.299922:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775786.299925:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775786.299927:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775786.299931:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775786.299936:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775786.299940:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775786.299944:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775786.299947:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775786.299951:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775786.299959:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061500. 00000020:00000040:0.0:1775775786.299964:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775786.299969:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775786.299970:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775786.299972:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775786.299976:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775786.299979:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775786.299986:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6c00. 00000100:00000001:0.0:1775775786.299991:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775786.299992:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775786.299998:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 57 secs later) 00000100:00100000:0.0:1775775786.299999:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061500 x1862022710365696/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775786.300017:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775786.300022:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775786.300027:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb8d992] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061500 x1862022710365696/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775786.300039:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775786.300044:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775786.300050:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775786.300054:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775786.300058:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775786.300063:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12679) 00000800:00000200:1.1:1775775786.300549:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775786.300552:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 57 secs later) 00000800:00000200:1.1:1775775786.300858:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775786.300860:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 57 secs later) 00010000:00000001:3.0:1775775786.427267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775786.427272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775786.427278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775786.427280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775786.427284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775786.427286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775786.427384:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775786.427390:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775786.427394:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775787.323370:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775787.323375:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775787.323377:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775787.323381:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775787.323384:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775787.323387:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775787.323389:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775787.323392:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775787.323394:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775787.451271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775787.451277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775787.451282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775787.451283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775787.451287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775787.451288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775787.451354:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775787.451359:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775787.451363:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775788.347222:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775788.347229:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775788.347231:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775788.347237:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775788.347241:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775788.347245:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775788.347248:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775788.347250:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775788.347253:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775788.475254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775788.475260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775788.475264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775788.475267:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775788.475268:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775788.475271:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775788.475273:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775788.475275:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775788.475277:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775788.475278:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775788.475281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775788.475281:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775788.475282:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775788.475283:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775788.475284:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775788.475285:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775788.475286:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775788.475287:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775788.475367:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775788.475372:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775788.475376:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775789.371288:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775789.371295:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775789.371297:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775789.371303:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775789.371307:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775789.371311:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775789.371314:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775789.371317:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775789.371320:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775789.499217:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775789.499226:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775789.499234:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775789.499236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775789.499240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775789.499241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775790.203256:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307345158, at 4307345216 00000100:00000010:0.0:1775775790.203271:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2120. 00000100:00000001:3.0:1775775790.395191:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775790.395196:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775790.395198:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775790.395202:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775790.395205:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775790.395208:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775790.395210:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775790.395212:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775790.395214:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775790.523260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775790.523267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775790.523280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775790.523282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775790.523286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775790.523288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775790.523311:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775790.523314:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775790.523316:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775791.419239:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775775791.419239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775791.419245:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775791.419247:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775791.419250:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:3.0:1775775791.419252:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775775791.419254:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775775791.419256:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775775791.419259:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775791.419259:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775791.419265:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775775791.419270:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775775791.419273:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12674 00000100:00000001:3.0:1775775791.419274:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775791.419279:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775791.419286:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775791.419296:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775791.419300:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775791.419304:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775791.419308:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775791.419312:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775791.419315:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775791.419319:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775791.419321:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775791.419324:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775791.419326:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775791.419329:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775791.419331:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775791.419335:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775791.419337:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775791.419339:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775791.419342:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775791.419345:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775791.419348:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775791.419351:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775791.419354:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775791.419358:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775791.419361:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775791.419364:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775791.419366:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775791.419384:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062a00. 00000020:00000040:0.0:1775775791.419387:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775791.419392:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775791.419394:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775791.419398:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775791.419400:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775791.419404:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775791.419412:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7800. 00000100:00000001:0.0:1775775791.419420:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775791.419429:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062a00 x1862022710365824/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775791.419441:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775791.419446:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775791.419452:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009753f9f6] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062a00 x1862022710365824/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775791.419477:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775791.419480:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775791.419488:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775791.419490:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775791.419492:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775791.419495:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775791.419498:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12674 00000400:00000200:0.0:1775775791.419503:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775791.419508:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775791.419511:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775791.419515:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775791.419520:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775791.419525:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775791.419528:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775791.419532:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775791.419543:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775791.419548:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775791.419553:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775791.419557:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775791.419563:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775791.419571:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775791.419575:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775791.419582:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775791.419586:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775791.419590:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775791.419597:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775791.419601:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775791.419606:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775791.419614:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775791.419619:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775791.419622:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775791.419627:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775791.419635:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775791.419639:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775791.419646:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062d80. 00000020:00000040:0.0:1775775791.419652:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775791.419658:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775791.419661:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775791.419665:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775791.419667:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775791.419671:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775791.419677:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6400. 00000100:00000001:0.0:1775775791.419683:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775791.419691:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062d80 x1862022710365952/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775791.419709:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775791.419713:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775791.419721:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009965f80b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062d80 x1862022710365952/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775791.419747:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775791.419751:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775791.419759:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775791.419762:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775791.419766:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775791.419770:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775791.419775:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12674 00000800:00000200:1.1:1775775791.419800:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000400:00000200:0.0:1775775791.419803:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000200:1.1:1775775791.419804:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000400:00000200:0.0:1775775791.419814:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775791.419820:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775791.419826:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775791.419831:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775791.419837:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775791.419842:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775791.419847:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775791.419849:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775791.419853:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775791.419856:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775791.419862:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775791.419868:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775791.419875:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775791.419879:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775791.419886:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775791.419892:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775791.419897:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775791.419902:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775791.419908:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775791.419913:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775791.419919:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775791.419925:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775791.419927:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775791.419930:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775791.419934:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775791.419938:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775791.419946:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062680. 00000020:00000040:0.0:1775775791.419951:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775791.419957:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775791.419961:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775791.419965:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775791.419968:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775791.419972:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775791.419979:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4400. 00000100:00000001:0.0:1775775791.419986:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775791.419995:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062680 x1862022710366080/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775791.420019:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775791.420024:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775791.420034:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000082c7985e] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062680 x1862022710366080/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775791.420061:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775791.420066:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775791.420074:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775791.420078:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775791.420088:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775791.420097:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12684) 00000800:00000200:1.1:1775775791.420305:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775791.420308:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1775775791.420533:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775791.420535:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00010000:00000001:3.0:1775775791.547201:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775791.547215:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775791.547223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775791.547224:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775791.547227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775791.547228:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775791.547285:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775791.547289:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775791.547292:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775792.443262:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775792.443267:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775792.443268:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775792.443272:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775792.443275:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775792.443278:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775792.443280:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775792.443283:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775792.443284:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775792.571273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775792.571279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775792.571285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775792.571286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775792.571291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775792.571292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775792.571359:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775792.571363:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775792.571365:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775793.467253:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775793.467261:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775793.467263:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775793.467270:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775793.467274:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775793.467277:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775793.467280:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775793.467283:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775793.467286:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775793.595256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775793.595263:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775793.595265:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775793.595269:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775793.595273:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775793.595276:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775793.595278:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775793.595281:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775793.595285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775793.595287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775793.595291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775793.595293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775793.595364:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775793.595367:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775793.595370:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775794.491222:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775794.491230:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775794.491231:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775794.491236:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775794.491239:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775794.491241:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775794.491243:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775794.491246:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775794.491248:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775794.619254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775794.619259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775794.619263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775794.619264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775794.619267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775794.619268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775795.323184:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307350278, at 4307350336 00000100:00000010:0.0:1775775795.323197:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e25a0. 00000100:00000001:3.0:1775775795.515245:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775795.515251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775795.515254:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775795.515259:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775795.515262:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775795.515267:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775795.515270:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775795.515272:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775795.515275:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775795.643256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775795.643261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775795.643267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775795.643268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775795.643272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775795.643273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775795.643296:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775795.643299:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775795.643302:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775796.539294:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775796.539306:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775796.539309:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775796.539314:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775796.539319:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775775796.539325:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1775775796.539329:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12679 00000100:00000001:3.0:1775775796.539331:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775796.539334:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775796.539339:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000400:00000200:2.0:1775775796.539340:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775775796.539344:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1775775796.539349:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:3.0:1775775796.539349:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775775796.539355:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1775775796.539355:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775796.539359:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775775796.539362:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775796.539364:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775796.539367:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775796.539372:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775796.539375:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775796.539379:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775796.539381:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775796.539385:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775796.539388:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775796.539391:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775796.539394:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775796.539397:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775796.539399:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775796.539401:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775796.539403:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775796.539406:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775796.539409:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775796.539412:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775796.539416:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775796.539419:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775796.539422:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775796.539425:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775796.539427:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775796.539435:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775775796.539439:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775796.539443:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775796.539445:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775796.539448:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775796.539450:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775796.539455:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775796.539462:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775775796.539470:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775796.539479:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710366208/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775796.539489:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775796.539493:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775796.539499:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710366208/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775796.539521:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775796.539524:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775796.539527:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775796.539529:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775796.539531:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775796.539533:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775796.539536:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12679 00000400:00000200:2.0:1775775796.539540:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775796.539544:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775796.539547:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775796.539551:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775796.539553:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775796.539556:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775796.539559:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775796.539563:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775796.539566:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775796.539568:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775796.539572:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775796.539575:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775796.539579:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775796.539587:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775796.539593:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775796.539598:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775796.539602:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775796.539606:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775796.539610:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775796.539615:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775796.539618:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775796.539625:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775796.539631:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775796.539635:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775796.539641:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775796.539646:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775796.539650:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775796.539657:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775775796.539662:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775796.539665:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775796.539668:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775796.539674:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775796.539676:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775796.539679:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775796.539686:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775775796.539691:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775796.539716:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710366336/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775796.539729:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775796.539733:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775796.539743:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710366336/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775796.539778:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775796.539783:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775796.539789:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775796.539794:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775796.539797:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775796.539802:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775796.539808:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12679 00000400:00000200:2.0:1775775796.539814:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775796.539821:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775796.539825:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775796.539830:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775796.539837:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775796.539841:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775796.539845:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775796.539850:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775796.539852:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775796.539855:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775796.539859:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775796.539863:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775796.539867:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775796.539875:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775796.539880:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775796.539886:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775796.539890:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775796.539893:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775796.539897:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775796.539902:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775796.539905:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775796.539910:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775796.539915:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775796.539919:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775796.539923:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775796.539927:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775796.539930:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775796.539935:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775775796.539940:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775796.539945:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775796.539948:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775796.539951:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775796.539954:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775796.539958:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775796.539958:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000010:2.0:1775775796.539965:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000800:00000200:1.1:1775775796.539965:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000100:00000001:2.0:1775775796.539972:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775796.539980:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710366464/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775796.539995:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775796.540000:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775796.540008:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710366464/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775796.540021:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775796.540025:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775796.540031:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775796.540037:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775796.540046:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775796.540054:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12689) 00000800:00000200:1.1:1775775796.540364:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775796.540366:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1775775796.540653:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775796.540656:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00010000:00000001:3.0:1775775796.667248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775796.667253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775796.667257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775796.667258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775796.667261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775796.667262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775796.667317:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775796.667320:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775796.667323:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.0:1775775797.371275:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775775797.371284:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775775797.371289:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000100:00000001:3.0:1775775797.563266:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775797.563272:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775797.563274:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775797.563286:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775797.563289:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775797.563292:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775797.563294:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775797.563296:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775797.563298:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775797.691257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775797.691263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775797.691269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775797.691271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775797.691275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775797.691276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775797.691349:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775797.691353:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775797.691358:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775798.587232:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775798.587237:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775798.587239:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775798.587243:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775798.587246:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775798.587249:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775798.587252:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775798.587254:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775798.587256:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775798.715245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775798.715250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775798.715254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775798.715257:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775798.715258:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775798.715261:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775798.715263:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775798.715265:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775798.715267:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775798.715268:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775798.715272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775798.715272:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775798.715273:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775798.715274:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775798.715275:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775798.715276:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775798.715278:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775798.715279:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775798.715343:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775798.715347:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775798.715351:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775799.611243:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775799.611248:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775799.611250:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775799.611255:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775799.611258:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775799.611261:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775799.611263:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775799.611265:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775799.611268:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775799.739228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775799.739234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775799.739238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775799.739239:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775799.739242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775799.739243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775800.443245:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307355398, at 4307355456 00000100:00000010:0.0:1775775800.443263:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2300. 00000100:00000001:3.0:1775775800.635217:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775800.635226:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775800.635229:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775800.635234:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775800.635238:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775800.635242:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775800.635245:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775800.635248:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775800.635251:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775800.763236:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775800.763240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775800.763244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775800.763245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775800.763248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775800.763249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775800.763268:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775800.763270:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775800.763272:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775801.659227:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1775775801.659228:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775775801.659233:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1775775801.659236:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775801.659236:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775801.659240:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1775775801.659241:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:2.0:1775775801.659243:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775801.659245:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775801.659248:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775775801.659249:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775801.659251:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775801.659254:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775775801.659255:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12684 00000100:00000001:3.0:1775775801.659258:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775775801.659265:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775801.659270:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775801.659272:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775801.659275:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775801.659278:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775801.659280:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775801.659282:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775801.659284:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775801.659286:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775801.659288:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775801.659290:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775801.659292:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775801.659293:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775801.659295:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775801.659297:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775801.659298:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775801.659299:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775801.659301:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775801.659303:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775801.659304:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775801.659308:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775801.659311:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775801.659314:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775801.659316:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775801.659318:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775801.659327:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775775801.659331:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775801.659335:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775801.659337:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775801.659340:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775801.659342:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775801.659346:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775801.659355:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775775801.659361:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775801.659370:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710366592/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775801.659381:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775801.659385:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775801.659389:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710366592/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775801.659411:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775801.659414:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775801.659417:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775801.659419:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775801.659421:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775801.659424:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775801.659427:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12684 00000400:00000200:2.0:1775775801.659432:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775801.659437:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775801.659441:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775801.659447:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775801.659451:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775801.659458:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775801.659463:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775801.659469:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775801.659471:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775801.659474:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775801.659477:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775801.659481:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775801.659485:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775801.659492:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775801.659496:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775801.659500:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775801.659503:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775801.659505:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775801.659508:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775801.659512:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775801.659517:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775801.659525:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775801.659529:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775801.659533:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775801.659539:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775801.659544:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775801.659549:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775801.659554:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775775801.659557:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775801.659559:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775801.659562:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775801.659564:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775801.659565:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775801.659567:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775801.659573:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775801.659577:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775801.659584:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710366720/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775801.659622:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775801.659624:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775801.659628:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710366720/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775801.659642:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775801.659645:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775801.659649:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775801.659651:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775801.659652:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775801.659657:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775801.659660:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12684 00000400:00000200:2.0:1775775801.659666:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775801.659673:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775801.659675:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775801.659679:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775801.659682:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775801.659685:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775801.659688:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775801.659690:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775801.659692:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775801.659696:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775801.659697:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775801.659700:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775801.659704:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775801.659709:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775801.659711:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775801.659715:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775801.659719:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775801.659724:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775801.659729:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775801.659733:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775801.659737:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775801.659740:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775801.659741:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775801.659743:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775801.659745:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775801.659751:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775801.659755:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775801.659759:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775775801.659763:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775801.659766:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775801.659767:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775801.659769:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775801.659770:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775801.659773:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775801.659777:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775775801.659782:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775801.659788:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710366848/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1775775801.659794:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775801.659799:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000100:00000001:2.0:1775775801.659800:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775801.659804:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775801.659811:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710366848/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775801.659819:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775801.659824:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775801.659830:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775801.659832:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775801.659837:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775801.659843:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12694) 00000800:00000200:1.1:1775775801.660117:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775801.660119:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775775801.660360:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775801.660362:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00010000:00000001:3.0:1775775801.787236:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775801.787242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775801.787248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775801.787250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775801.787254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775801.787255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775801.787310:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775801.787313:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775801.787316:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775802.683211:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775802.683218:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775802.683220:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775802.683225:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775802.683229:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775802.683232:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775802.683235:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775802.683238:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775802.683240:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775802.811267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775802.811273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775802.811278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775802.811279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775802.811282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775802.811283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775802.811364:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775802.811369:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775802.811373:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775803.707208:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775803.707213:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775803.707214:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775803.707218:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775803.707221:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775803.707224:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775803.707227:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775803.707228:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775803.707230:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775803.835228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775803.835234:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775803.835237:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775803.835240:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775803.835245:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775803.835255:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775803.835257:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775803.835260:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775803.835267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775803.835269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775803.835273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775803.835274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775803.835345:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775803.835350:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775803.835353:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775804.731253:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775804.731259:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775804.731262:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775804.731268:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775804.731272:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775804.731275:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775804.731286:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775804.731289:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775804.731293:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775804.859237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775804.859243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775804.859250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775804.859251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775804.859256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775804.859257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775805.563216:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307360518, at 4307360576 00000100:00000010:0.0:1775775805.563230:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2de0. 00000100:00000001:3.0:1775775805.755223:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775805.755230:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775805.755232:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775805.755239:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775805.755243:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775805.755247:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775805.755250:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775805.755253:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775805.755255:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775805.883246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775805.883252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775805.883256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775805.883257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775805.883260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775805.883261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775805.883286:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775805.883289:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775805.883293:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775806.779272:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1775775806.779278:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775775806.779279:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775806.779282:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775806.779291:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:2.0:1775775806.779294:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775806.779296:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775806.779298:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775775806.779301:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775775806.779303:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775806.779305:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775806.779309:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775775806.779312:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775775806.779313:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775806.779322:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12689 00000400:00000200:2.0:1775775806.779334:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775806.779341:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775806.779345:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775806.779350:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775806.779355:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775806.779357:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775806.779359:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775806.779364:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775806.779366:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775806.779369:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775806.779372:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775806.779375:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775806.779377:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775806.779380:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775806.779382:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775806.779385:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775806.779387:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775806.779391:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775806.779394:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775806.779397:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775806.779401:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775806.779405:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775806.779408:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775806.779410:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775806.779413:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775806.779423:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775775806.779427:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775806.779431:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775806.779433:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775806.779436:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775806.779439:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775806.779443:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775806.779452:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775775806.779458:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775806.779467:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710366976/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775806.779478:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775806.779483:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775806.779488:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710366976/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775806.779514:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775806.779517:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775806.779547:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775806.779548:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775806.779550:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775806.779552:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775806.779556:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12689 00000400:00000200:2.0:1775775806.779560:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775806.779564:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775806.779567:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775806.779571:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775806.779574:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775806.779577:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775806.779579:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775806.779582:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775806.779584:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775806.779586:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775806.779588:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775806.779591:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775806.779594:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775806.779601:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775806.779606:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775806.779612:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775806.779616:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775806.779619:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775806.779628:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775806.779637:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775806.779645:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775806.779655:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775806.779660:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775806.779665:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775806.779673:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775806.779684:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775806.779689:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775806.779698:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775775806.779703:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775806.779708:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775806.779713:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775806.779720:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775806.779723:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775806.779729:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775806.779737:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775775806.779745:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775806.779757:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710367104/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775806.779811:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775806.779819:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775806.779835:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710367104/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775806.779853:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775806.779859:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775806.779864:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775806.779866:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775806.779872:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775806.779881:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775806.779887:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12689 00000400:00000200:2.0:1775775806.779895:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775806.779905:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775806.779911:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775806.779919:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775806.779926:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775806.779933:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775806.779939:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775806.779946:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775806.779948:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775806.779951:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775806.779956:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775806.779965:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775806.779973:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775806.779985:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775806.779989:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775806.779991:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775806.779993:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775806.779996:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775806.779999:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775806.780002:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775806.780004:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775806.780008:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775806.780010:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775806.780012:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775806.780015:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775806.780018:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775806.780020:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775806.780024:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775775806.780027:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775806.780030:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775806.780031:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775806.780034:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775806.780035:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775806.780037:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775806.780041:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775775806.780045:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775806.780049:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710367232/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775806.780057:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775806.780060:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775806.780064:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710367232/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775806.780073:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775806.780077:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775806.780081:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775806.780084:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775806.780088:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775806.780093:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12699) 00000800:00000200:1.1:1775775806.780313:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775806.780316:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775775806.780854:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775806.780857:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775775806.781238:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775806.781241:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00010000:00000001:3.0:1775775806.907255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775806.907262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775806.907268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775806.907270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775806.907279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775806.907281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775806.907378:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775806.907384:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775806.907390:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775807.803215:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775807.803220:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775807.803222:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775807.803227:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775807.803230:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775807.803232:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775807.803234:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775807.803236:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775807.803238:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775807.931246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775807.931251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775807.931255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775807.931256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775807.931259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775807.931260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775807.931336:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775807.931340:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775807.931344:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775808.827237:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775808.827244:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775808.827246:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775808.827251:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775808.827254:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775808.827257:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775808.827259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775808.827261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775808.827263:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775808.955226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775808.955232:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775808.955235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775808.955238:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775808.955240:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775808.955243:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775808.955245:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775808.955246:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775808.955247:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775808.955249:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775808.955252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775808.955254:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775808.955255:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775808.955258:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775808.955260:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775808.955261:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775808.955270:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775808.955272:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775808.955331:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775808.955335:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775808.955338:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775809.851203:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775809.851208:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775809.851210:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775809.851214:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775809.851216:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775809.851220:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775809.851222:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775809.851224:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775809.851226:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775809.979215:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775809.979220:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775809.979225:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775809.979226:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775809.979228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775809.979229:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775810.683229:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307365639, at 4307365696 00000100:00000010:0.0:1775775810.683245:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2660. 00000100:00000001:3.0:1775775810.875238:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775810.875244:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775810.875245:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775810.875249:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775810.875252:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775810.875256:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775810.875258:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775810.875260:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775810.875262:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775811.003226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775811.003238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775811.003245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775811.003247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775811.003251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775811.003252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775811.003277:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775811.003280:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775811.003283:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775811.899237:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775811.899244:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775811.899247:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775811.899254:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775811.899258:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775811.899264:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775811.899268:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775811.899272:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775811.899277:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775811.899480:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775811.899484:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775811.899486:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775811.899493:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491a680 x1862022710367360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.899502:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775811.899503:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775811.899507:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775811.899509:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775811.899511:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775811.899515:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710367360:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775811.899519:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775811.899523:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775811.899525:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775811.899528:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775811.899532:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775811.899535:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775811.899540:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775811.899544:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775775811.899551:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775775811.899557:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710367360, portal 25 00000100:00000001:0.0:1775775811.899560:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775811.899562:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684368512 : -123823025183104 : ffff8f623491a680) 00000100:00000040:0.0:1775775811.899566:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491a680 x1862022710367360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.899574:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775811.899577:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775811.899580:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775775811.899583:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710367360, offset 0 00000400:00000200:0.0:1775775811.899589:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775811.899597:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775811.899603:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775811.899609:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775811.899613:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775811.899618:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775811.899621:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775811.899625:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453422:453422:256:4294967295] 192.168.201.115@tcp LPNI seq info [453347:453347:8:4294967295] 00000400:00000200:0.0:1775775811.899630:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775811.899640:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775811.899646:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775811.899704:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775775811.899712:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775811.899715:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775775811.899721:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775811.899724:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1775775811.899729:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775811.899732:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775775811.899738:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775811.899742:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775811.899746:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775775811.899748:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775775811.899751:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775811.899754:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491a680 x1862022710367360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.899766:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775811.899769:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491a680 x1862022710367360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775836 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.899774:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775811.899775:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775811.899779:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775811.899783:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775811.899784:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307370758 00000100:00000010:0.0:1775775811.899788:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775811.899794:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775811.899799:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307370758 00000100:00000010:0.0:1775775811.899801:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3600. 00000100:00000001:0.0:1775775811.899803:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775811.899805:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775811.899807:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775811.899809:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775811.899811:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775811/real 1775775811] req@ffff8f623491a680 x1862022710367360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775811.899818:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775811.899819:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775811.899821:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775775811.899822:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775775811.899824:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775775811.899825:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775811.899827:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491a680 x1862022710367360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775811.899833:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491a680 x1862022710367360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.899837:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775811.899838:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775811.899839:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775811.899841:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775811.899843:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491a680 x1862022710367360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.899847:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775811.899850:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491a680 x1862022710367360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.899854:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775811.899856:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775811.899857:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775811.899859:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775811.899861:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775811.899862:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775811.899865:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775811.899868:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775811.899869:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775811.899872:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491a680 x1862022710367360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775811.899877:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710367360:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775811.899879:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775811.899881:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775811.899882:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491a680 x1862022710367360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.899886:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775811.899888:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775811.899890:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775811.899892:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775811.899893:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775811.899895:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775811.899896:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775811.899898:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4dc00. 02000000:00000001:0.0:1775775811.899900:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775811.899902:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775811.899903:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491a680. 00000100:00000001:0.0:1775775811.899905:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775811.899906:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775811.899907:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775811.899913:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775811.899914:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775811.899915:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775811.899916:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775811.899917:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775811.899919:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491a300 x1862022710367488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.899925:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775811.899926:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775811.899927:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775811.899928:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775811.899930:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775811.899931:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710367488:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775811.899933:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775811.899934:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775811.899935:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775811.899936:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775811.899938:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775811.899939:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775811.899941:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775811.899943:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775775811.899945:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775775811.899947:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710367488, portal 10 00000100:00000001:0.0:1775775811.899949:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775811.899950:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684367616 : -123823025184000 : ffff8f623491a300) 00000100:00000040:0.0:1775775811.899954:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491a300 x1862022710367488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.899958:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775811.899960:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775811.899961:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775775811.899963:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710367488, offset 0 00000400:00000200:0.0:1775775811.899965:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775811.899968:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775811.899970:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775811.899972:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775811.899974:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775811.899976:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775811.899977:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775811.899980:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453423:453423:256:4294967295] 192.168.201.115@tcp LPNI seq info [453348:453348:8:4294967295] 00000400:00000200:0.0:1775775811.899983:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775811.899987:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775811.899990:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775811.899993:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775775811.899995:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775811.899997:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775775811.900000:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775811.900002:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1775775811.900005:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775811.900006:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775775811.900008:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775811.900010:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775811.900012:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775775811.900013:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775775811.900015:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775811.900017:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491a300 x1862022710367488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.900022:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775811.900024:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491a300 x1862022710367488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775836 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.900028:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775811.900029:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775811.900030:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775811.900033:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775811.900035:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775811.900036:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307366923 00000100:00000001:0.0:1775775811.900038:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307370758 00000100:00000001:0.0:1775775811.900040:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307366923 00000100:00000001:0.0:1775775811.900041:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307370758 00000100:00000001:0.0:1775775811.900043:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775811.900044:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775811.900046:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775811.900048:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775811/real 1775775811] req@ffff8f623491a300 x1862022710367488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775811.900055:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775811.900057:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775811.900059:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775775811.900061:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775775811.900063:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775775811.900065:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775811.900068:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491a300 x1862022710367488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775811.900077:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491a300 x1862022710367488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.900084:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775811.900086:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775811.900088:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775811.900089:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775811.900092:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491a300 x1862022710367488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.900099:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775811.900103:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491a300 x1862022710367488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.900110:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775811.900112:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775811.900114:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775811.900115:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775811.900117:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775811.900121:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775811.900124:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775811.900127:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775811.900129:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775811.900173:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491a300 x1862022710367488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775811.900182:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710367488:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775811.900186:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775811.900189:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234919c00 x1862022710367616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.900195:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775811.900197:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775811.900199:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775811.900201:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775811.900203:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775811.900206:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234919c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710367616:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775811.900209:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775811.900211:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775811.900212:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775811.900214:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775811.900218:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775811.900220:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775811.900223:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775811.900226:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775775811.900229:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775775811.900232:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710367616, portal 10 00000100:00000001:0.0:1775775811.900234:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775811.900235:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684365824 : -123823025185792 : ffff8f6234919c00) 00000100:00000040:0.0:1775775811.900240:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234919c00 x1862022710367616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.900247:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775811.900249:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775811.900251:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775775811.900254:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710367616, offset 0 00000400:00000200:0.0:1775775811.900258:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775811.900262:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775811.900266:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775811.900269:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775811.900271:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775811.900274:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775811.900275:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775811.900278:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453424:453424:256:4294967295] 192.168.201.115@tcp LPNI seq info [453349:453349:8:4294967295] 00000400:00000200:0.0:1775775811.900281:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775811.900285:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775811.900289:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775811.900291:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775775811.900294:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775811.900296:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775775811.900299:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775811.900301:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1775775811.900303:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775811.900305:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775775811.900307:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775811.900309:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775811.900311:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775775811.900313:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775775811.900314:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775811.900317:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234919c00 x1862022710367616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775836 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.900322:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775811.900324:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234919c00 x1862022710367616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775836 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.900328:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775811.900330:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775811.900332:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775811.900334:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775811.900336:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775811.900337:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307366924 00000100:00000001:0.0:1775775811.900339:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307370758 00000100:00000001:0.0:1775775811.900341:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307366924 00000100:00000001:0.0:1775775811.900342:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307370758 00000100:00000001:0.0:1775775811.900343:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775811.900345:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775811.900346:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775811.900348:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775811/real 1775775811] req@ffff8f6234919c00 x1862022710367616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775811.900353:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775811.900354:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775811.900355:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775775811.900357:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775775811.900358:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775775811.900359:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775811.900361:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234919c00 x1862022710367616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775811.900366:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234919c00 x1862022710367616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.900370:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775811.900371:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775811.900372:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775811.900373:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775811.900375:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234919c00 x1862022710367616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.900378:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775811.900381:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234919c00 x1862022710367616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.900386:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775811.900388:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775811.900389:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775811.900389:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775811.900390:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775811.900392:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775811.900393:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775811.900395:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775811.900396:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775811.900399:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234919c00 x1862022710367616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775811.900404:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234919c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710367616:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775811.900406:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775811.900407:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775811.900409:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491a300 x1862022710367488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.900413:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775811.900414:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775811.900415:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775811.900417:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775811.900417:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775811.900418:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775811.900419:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775811.900421:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4c400. 02000000:00000001:0.0:1775775811.900422:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775811.900423:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775811.900424:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491a300. 00000100:00000001:0.0:1775775811.900426:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775811.900427:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775811.900445:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775811.900448:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234919c00 x1862022710367616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775836 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775811.900453:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775811.900454:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775811.900456:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775811.900458:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775811.900459:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775811.900461:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775811.900462:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775811.900464:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:0.0:1775775811.900466:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775811.900467:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775811.900469:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234919c00. 00000100:00000001:0.0:1775775811.900471:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775811.900472:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775811.900474:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775811.900478:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775811.900479:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775811.900481:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775811.900483:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775812.027270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775812.027277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775812.027283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775812.027285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775812.027289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775812.027291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775812.027357:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775812.027361:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775812.027363:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775812.923244:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775812.923251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775812.923253:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775812.923259:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775812.923262:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775812.923267:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775812.923269:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775812.923272:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775812.923275:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775813.051208:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775813.051213:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775813.051217:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775813.051218:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775813.051221:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775813.051222:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775813.051285:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775813.051289:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775813.051294:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775813.947253:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775813.947260:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775813.947262:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775813.947267:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775813.947271:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775813.947275:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775813.947278:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775813.947280:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775813.947283:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775814.075278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775814.075286:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775814.075288:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775814.075291:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775814.075297:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775814.075299:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775814.075301:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775814.075303:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775814.075307:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775814.075308:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775814.075311:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775814.075312:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775814.075413:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775814.075419:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775814.075424:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775814.971232:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775814.971239:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775814.971241:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775814.971247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775814.971251:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775814.971255:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775814.971258:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775814.971260:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775814.971263:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775815.099247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775815.099252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775815.099258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775815.099259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775815.099270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775815.099271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775815.803254:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307370758, at 4307370816 00000100:00000010:0.0:1775775815.803271:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00010000:00000001:3.0:1775775816.123278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775816.123284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775816.123290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775816.123291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775816.123294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775816.123295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775816.123322:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775816.123324:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775816.123326:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775817.019282:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775817.019290:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775817.019294:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775817.019299:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775817.019304:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775817.019311:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12699 00000400:00000200:3.0:1775775817.019321:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775817.019327:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775817.019331:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775817.019334:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775817.019337:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775817.019353:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775817.019356:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775817.019360:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775817.019362:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775817.019365:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775817.019367:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775817.019370:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775817.019373:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775817.019377:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775817.019379:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775817.019381:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775817.019383:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775817.019386:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775817.019390:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775817.019392:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775817.019396:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775817.019399:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775817.019403:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775817.019405:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775817.019407:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775817.019416:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775775817.019420:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775817.019424:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775817.019426:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775817.019429:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775817.019431:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775817.019436:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775817.019444:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000100:00000001:3.0:1775775817.019451:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775817.019461:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710367744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775817.019472:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775817.019477:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775817.019482:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710367744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775817.019507:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775817.019510:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775817.019514:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775817.019516:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775817.019517:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775817.019520:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775817.019522:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12699 00000400:00000200:3.0:1775775817.019526:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775817.019529:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775817.019533:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775817.019536:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775817.019539:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775817.019542:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775817.019544:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:0.0:1775775817.019547:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1775775817.019547:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775817.019550:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:0.0:1775775817.019552:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1775775817.019553:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775775817.019554:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1775775817.019558:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000040:0.0:1775775817.019563:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71180 x1862022710367744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775775817.019563:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775817.019567:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775817.019576:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775817.019577:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775817.019580:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775817.019581:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775775817.019584:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1775775817.019585:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:0.0:1775775817.019588:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775775817.019589:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:0.0:1775775817.019592:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775775817.019593:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00100000:0.0:1775775817.019597:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710367744:192.168.201.115@tcp:250:kworker.0 00000100:00080000:3.0:1775775817.019601:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775817.019607:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775817.019608:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775775817.019611:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775817.019613:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775817.019615:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775817.019617:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1775775817.019618:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:0.0:1775775817.019621:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:3.0:1775775817.019622:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775817.019626:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775817.019627:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775817.019633:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1775775817.019634:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775817.019638:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000008:00000001:3.0:1775775817.019639:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775817.019642:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775775817.019644:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000010:3.0:1775775817.019647:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000100:00000200:0.0:1775775817.019650:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710367744, portal 25 00000020:00000040:3.0:1775775817.019650:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000001:0.0:1775775817.019654:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1775775817.019654:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775817.019657:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775817.019658:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849008512 : -123823860543104 : ffff8f6202c71180) 00000100:00000001:3.0:1775775817.019663:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775817.019666:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:0.0:1775775817.019667:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71180 x1862022710367744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775775817.019670:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775817.019677:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:0.0:1775775817.019678:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775817.019682:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775775817.019683:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775775817.019685:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775775817.019689:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710367744, offset 0 00000100:00100000:3.0:1775775817.019693:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710367872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775775817.019697:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775817.019706:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775775817.019707:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775817.019711:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775817.019713:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000040:3.0:1775775817.019717:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710367872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775775817.019718:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775817.019724:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775817.019730:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775817.019734:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775817.019741:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453425:453425:256:4294967295] 192.168.201.115@tcp LPNI seq info [453350:453350:8:4294967295] 00000400:00000200:0.0:1775775817.019752:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775775817.019754:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775817.019759:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775775817.019764:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00080000:3.0:1775775817.019767:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775817.019770:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:0.0:1775775817.019771:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00080000:3.0:1775775817.019774:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000010:0.0:1775775817.019777:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000100:00000001:3.0:1775775817.019778:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775817.019783:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12699 00000800:00000200:0.0:1775775817.019786:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775817.019791:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000400:00000200:3.0:1775775817.019793:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000200:0.1:1775775817.019798:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000400:00000200:3.0:1775775817.019800:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000800:00000200:0.1:1775775817.019802:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000100:00080000:3.0:1775775817.019804:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775775817.019809:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775817.019813:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000100:00000040:3.0:1775775817.019813:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775817.019817:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775775817.019819:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:3.0:1775775817.019819:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775817.019824:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775817.019828:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:0.0:1775775817.019829:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000020:00000040:3.0:1775775817.019830:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775817.019833:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:0.0:1775775817.019834:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000020:00000040:3.0:1775775817.019835:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000400:00000010:0.0:1775775817.019839:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000020:00000001:3.0:1775775817.019839:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775817.019844:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000001:0.0:1775775817.019846:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1775775817.019848:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775817.019851:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:0.0:1775775817.019852:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71180 x1862022710367744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775775817.019857:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775817.019863:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:0.0:1775775817.019866:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000040:3.0:1775775817.019867:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00000040:0.0:1775775817.019871:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71180 x1862022710367744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775842 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775775817.019875:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775817.019882:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775817.019884:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775817.019888:0:2411:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1775775817.019889:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775775817.019893:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775817.019893:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775817.019899:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775817.019902:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775817.019905:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307375878 00000100:00000010:0.0:1775775817.019909:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775817.019915:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775817.019920:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307375878 00000100:00000010:0.0:1775775817.019922:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3ba0. 00000100:00000001:0.0:1775775817.019924:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775817.019927:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775817.019929:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775817.019931:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1775775817.019931:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00100000:0.0:1775775817.019935:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775817/real 1775775817] req@ffff8f6202c71180 x1862022710367744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775775817.019935:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775817.019942:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775817.019947:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1775775817.019951:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775817.019953:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000010:3.0:1775775817.019954:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000020:00000040:3.0:1775775817.019957:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775817.019959:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000010:0.0:1775775817.019962:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 02000000:00000001:3.0:1775775817.019962:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775817.019967:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000100:00000001:3.0:1775775817.019967:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775817.019970:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:0.0:1775775817.019971:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775775817.019974:0:2411:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1775775817.019975:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775775817.019978:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71180 x1862022710367744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1775775817.019981:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000200:0.0:1775775817.019986:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71180 x1862022710367744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775817.019988:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775817.019996:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775817.019999:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1775775817.019999:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710368000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775817.020001:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775817.020003:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775817.020010:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71180 x1862022710367744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775817.020019:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775817.020021:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775817.020024:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775817.020027:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71180 x1862022710367744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775817.020031:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775817.020036:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775817.020038:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775817.020039:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775775817.020041:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710368000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775817.020042:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775817.020044:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775817.020050:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775817.020060:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:3.0:1775775817.020061:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775817.020064:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1775775817.020066:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775817.020073:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71180 x1862022710367744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775775817.020073:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775817.020078:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775817.020087:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00100000:0.0:1775775817.020095:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710367744:192.168.201.115@tcp:250:kworker.0 00000100:00000040:3.0:1775775817.020095:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12709) 00000100:00000001:0.0:1775775817.020105:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775817.020108:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775817.020111:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71180 x1862022710367744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775817.020118:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775817.020121:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775817.020123:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775817.020126:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775817.020128:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775817.020129:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775817.020132:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775817.020167:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35000. 02000000:00000001:0.0:1775775817.020172:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775817.020174:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775817.020175:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71180. 00000100:00000001:0.0:1775775817.020178:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775817.020179:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775817.020182:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775817.020189:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775817.020190:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775817.020192:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775817.020194:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775817.020196:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775817.020201:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70700 x1862022710367872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775817.020208:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775817.020210:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775817.020211:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775817.020213:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775817.020216:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775817.020218:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710367872:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775817.020222:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775817.020224:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775817.020225:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775817.020227:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775817.020230:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775817.020232:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775817.020237:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775817.020239:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775775817.020243:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775775817.020246:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710367872, portal 10 00000100:00000001:0.0:1775775817.020248:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775817.020249:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849005824 : -123823860545792 : ffff8f6202c70700) 00000100:00000040:0.0:1775775817.020256:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70700 x1862022710367872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775817.020263:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775817.020266:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775817.020268:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775775817.020271:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710367872, offset 0 00000400:00000200:0.0:1775775817.020275:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775817.020279:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775817.020283:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775817.020286:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775817.020288:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775817.020290:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775817.020291:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775817.020294:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453426:453426:256:4294967295] 192.168.201.115@tcp LPNI seq info [453351:453351:8:4294967295] 00000400:00000200:0.0:1775775817.020297:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775817.020302:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775817.020307:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775817.020311:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775775817.020315:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775817.020318:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775775817.020323:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775817.020325:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1775775817.020329:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775817.020331:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775775817.020335:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775817.020338:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775817.020356:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775775817.020358:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775775817.020361:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775817.020365:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70700 x1862022710367872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775817.020374:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775817.020376:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70700 x1862022710367872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775842 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775817.020381:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775817.020382:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775817.020385:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775817.020388:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775817.020390:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775817.020391:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307372044 00000100:00000001:0.0:1775775817.020393:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307375878 00000100:00000001:0.0:1775775817.020395:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307372044 00000100:00000001:0.0:1775775817.020397:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307375878 00000100:00000001:0.0:1775775817.020398:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775817.020399:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775817.020401:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775817.020403:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775817/real 1775775817] req@ffff8f6202c70700 x1862022710367872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775817.020408:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775817.020409:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775817.020410:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775775817.020412:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775775817.020413:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775775817.020414:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775817.020417:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70700 x1862022710367872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775817.020422:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70700 x1862022710367872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775817.020427:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775817.020429:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775817.020430:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775817.020432:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775817.020435:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70700 x1862022710367872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775817.020442:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775817.020446:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70700 x1862022710367872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775817.020453:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775817.020456:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775817.020457:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775817.020459:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775817.020460:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775817.020465:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775817.020468:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775817.020472:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775817.020475:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775817.020479:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70700 x1862022710367872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775817.020487:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710367872:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775817.020490:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775817.020492:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70a80 x1862022710368000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775817.020501:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775817.020502:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775817.020504:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775817.020506:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775817.020507:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775817.020509:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710368000:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775817.020510:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775817.020512:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775817.020513:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775817.020514:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775817.020517:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775817.020518:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775817.020520:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775817.020522:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775775817.020524:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775775817.020526:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710368000, portal 10 00000100:00000001:0.0:1775775817.020527:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775817.020529:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849006720 : -123823860544896 : ffff8f6202c70a80) 00000100:00000040:0.0:1775775817.020533:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70a80 x1862022710368000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775817.020537:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775817.020539:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775817.020540:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775775817.020541:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710368000, offset 0 00000400:00000200:0.0:1775775817.020544:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775817.020546:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775817.020549:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775817.020551:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775817.020552:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775817.020554:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775817.020556:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775817.020559:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453427:453427:256:4294967295] 192.168.201.115@tcp LPNI seq info [453352:453352:8:4294967295] 00000400:00000200:0.0:1775775817.020561:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775817.020566:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775817.020569:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775817.020571:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775775817.020574:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775817.020575:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1775775817.020578:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775817.020580:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1775775817.020582:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775817.020584:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775775817.020586:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775817.020588:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775817.020590:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775775817.020591:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775775817.020593:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775817.020595:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70a80 x1862022710368000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775842 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775817.020600:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775817.020602:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70a80 x1862022710368000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775842 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775817.020606:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775817.020607:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775817.020609:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775817.020611:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775817.020612:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775817.020614:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307372044 00000100:00000001:0.0:1775775817.020616:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307375878 00000100:00000001:0.0:1775775817.020617:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307372044 00000100:00000001:0.0:1775775817.020618:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307375878 00000100:00000001:0.0:1775775817.020620:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775817.020622:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775817.020623:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775817.020625:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775817/real 1775775817] req@ffff8f6202c70a80 x1862022710368000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775817.020630:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775817.020631:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775817.020632:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775775817.020633:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775775817.020634:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775775817.020636:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775817.020637:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70a80 x1862022710368000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775817.020643:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70a80 x1862022710368000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775817.020647:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775817.020648:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775817.020649:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775817.020650:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775817.020652:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70a80 x1862022710368000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775817.020655:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775817.020658:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70a80 x1862022710368000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775817.020662:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775817.020663:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775817.020664:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775817.020665:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775817.020666:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775817.020667:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775817.020669:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775817.020672:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775817.020673:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775817.020675:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70a80 x1862022710368000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775817.020680:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710368000:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775817.020682:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775817.020684:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775817.020685:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70700 x1862022710367872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775817.020689:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775817.020690:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775817.020691:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775817.020692:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775817.020693:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775817.020694:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775817.020695:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775817.020696:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37c00. 02000000:00000001:0.0:1775775817.020698:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775817.020699:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775817.020699:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70700. 00000100:00000001:0.0:1775775817.020701:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775817.020702:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775817.020703:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775817.020704:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70a80 x1862022710368000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775842 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775817.020709:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775817.020709:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775817.020710:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775817.020712:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775817.020712:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775817.020713:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775817.020714:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775817.020715:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37800. 02000000:00000001:0.0:1775775817.020717:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775817.020717:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775817.020718:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70a80. 00000100:00000001:0.0:1775775817.020720:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775817.020720:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775817.020721:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775817.020724:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775817.020725:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775817.020726:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775817.020727:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775817.147226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775817.147231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775817.147236:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775817.147237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775817.147240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775817.147241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775817.147376:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775817.147381:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775817.147384:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775818.171268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775818.171274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775818.171280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775818.171281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775818.171285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775818.171287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775818.171375:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775818.171382:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775818.171388:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775819.195226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775819.195231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775819.195235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775819.195238:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775819.195239:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775819.195242:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775819.195244:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775819.195246:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775819.195247:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775819.195249:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775819.195252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775819.195253:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775819.195254:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775819.195255:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775819.195256:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775819.195257:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775819.195258:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775819.195259:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775819.195375:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775819.195381:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775819.195386:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775820.219328:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775820.219334:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775820.219340:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775820.219341:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775820.219344:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775820.219345:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775820.923227:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307375878, at 4307375936 00000100:00000010:0.0:1775775820.923245:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00010000:00000001:3.0:1775775821.243247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775821.243254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775821.243260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775821.243261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775821.243284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775821.243286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775821.243317:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775821.243321:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775821.243326:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775775822.139676:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775822.139683:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775822.139687:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775822.139703:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234919f80 x1862022710368128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.139758:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775822.139764:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775822.139770:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775822.139778:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775822.139784:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775822.139794:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234919f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710368128:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775822.139808:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775822.139814:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775822.139819:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775822.139823:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775822.139834:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775822.139841:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775822.139850:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775822.139855:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775775822.139863:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775775822.139872:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710368128, portal 25 00000100:00000001:0.0:1775775822.139876:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775822.139879:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684366720 : -123823025184896 : ffff8f6234919f80) 00000100:00000040:0.0:1775775822.139890:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234919f80 x1862022710368128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775847 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.139925:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775822.139931:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775822.139941:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775775822.139949:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710368128, offset 0 00000400:00000200:0.0:1775775822.139962:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775822.139975:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775822.139993:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775822.140002:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775822.140011:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775822.140020:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775822.140024:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775822.140032:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453428:453428:256:4294967295] 192.168.201.115@tcp LPNI seq info [453353:453353:8:4294967295] 00000400:00000200:0.0:1775775822.140038:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775822.140061:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775822.140080:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775822.140092:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775775822.140102:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775822.140107:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775775822.140115:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775822.140123:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775775822.140208:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775822.140211:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775775822.140217:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775822.140223:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775822.140228:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775775822.140231:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775775822.140234:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775822.140237:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234919f80 x1862022710368128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775847 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140245:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775822.140248:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234919f80 x1862022710368128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775847 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140264:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775822.140266:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775822.140270:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775822.140273:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775822.140275:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307380999 00000100:00000010:0.0:1775775822.140280:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775822.140286:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775822.140292:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307380999 00000100:00000010:0.0:1775775822.140295:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3ba0. 00000100:00000001:0.0:1775775822.140297:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775822.140299:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775822.140302:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775822.140304:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775822.140308:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775822/real 1775775822] req@ffff8f6234919f80 x1862022710368128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775822.140318:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775822.140320:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775822.140323:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775775822.140325:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775775822.140327:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775775822.140330:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775822.140333:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234919f80 x1862022710368128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775822.140343:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234919f80 x1862022710368128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140350:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775822.140352:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775822.140355:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775822.140356:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775822.140360:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234919f80 x1862022710368128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140369:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775822.140373:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234919f80 x1862022710368128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140378:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775822.140380:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775822.140381:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775822.140382:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775822.140384:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775822.140386:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775822.140389:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775822.140393:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775822.140395:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775822.140401:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234919f80 x1862022710368128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775822.140410:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234919f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710368128:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775822.140415:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775822.140417:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775822.140420:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234919f80 x1862022710368128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140427:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775822.140431:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775822.140434:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775822.140437:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775822.140440:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775822.140442:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775822.140445:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775822.140447:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4d000. 02000000:00000001:0.0:1775775822.140451:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775822.140454:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775822.140456:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234919f80. 00000100:00000001:0.0:1775775822.140460:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775822.140462:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775822.140465:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775822.140472:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775822.140474:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775822.140477:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775822.140479:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775822.140480:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775822.140485:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918380 x1862022710368256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140494:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775822.140495:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775822.140498:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775822.140500:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775822.140502:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775822.140504:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710368256:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775822.140507:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775822.140508:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775822.140510:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775822.140511:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775822.140513:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775822.140515:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775822.140518:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775822.140520:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775775822.140523:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775775822.140525:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710368256, portal 10 00000100:00000001:0.0:1775775822.140526:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775822.140528:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684359552 : -123823025192064 : ffff8f6234918380) 00000100:00000040:0.0:1775775822.140532:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918380 x1862022710368256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775847 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140537:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775822.140541:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775822.140543:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775775822.140545:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710368256, offset 0 00000400:00000200:0.0:1775775822.140548:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775822.140551:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775822.140556:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775822.140559:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775822.140562:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775822.140564:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775822.140566:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775822.140570:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453429:453429:256:4294967295] 192.168.201.115@tcp LPNI seq info [453354:453354:8:4294967295] 00000400:00000200:0.0:1775775822.140573:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775822.140578:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775822.140581:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775822.140584:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775775822.140587:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775822.140589:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775775822.140592:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775822.140594:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775775822.140597:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775822.140598:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775775822.140601:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775822.140603:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775822.140605:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775775822.140607:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775775822.140608:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775822.140610:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918380 x1862022710368256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775847 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140616:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775822.140618:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918380 x1862022710368256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775847 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140623:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775822.140624:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775822.140626:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775822.140630:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775822.140632:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775822.140633:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307377164 00000100:00000001:0.0:1775775822.140635:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307380999 00000100:00000001:0.0:1775775822.140637:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307377164 00000100:00000001:0.0:1775775822.140639:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307380999 00000100:00000001:0.0:1775775822.140640:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775822.140642:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775822.140644:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775822.140646:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775822/real 1775775822] req@ffff8f6234918380 x1862022710368256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775822.140651:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775822.140652:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775822.140654:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775775822.140656:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775775822.140657:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775775822.140660:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775822.140662:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918380 x1862022710368256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775822.140669:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918380 x1862022710368256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140675:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775822.140678:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775822.140679:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775822.140680:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775822.140683:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918380 x1862022710368256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140687:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775822.140690:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918380 x1862022710368256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140694:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775822.140696:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775822.140697:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775822.140698:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775822.140699:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775822.140702:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775822.140704:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775822.140706:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775822.140707:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775822.140710:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918380 x1862022710368256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775822.140714:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710368256:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775822.140717:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775822.140719:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918e00 x1862022710368384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140723:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775822.140724:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775822.140726:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775822.140727:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775822.140729:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775822.140731:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710368384:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775822.140733:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775822.140734:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775822.140735:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775822.140736:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775822.140738:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775822.140739:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775822.140741:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775822.140743:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775775822.140745:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775775822.140747:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710368384, portal 10 00000100:00000001:0.0:1775775822.140748:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775822.140749:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684362240 : -123823025189376 : ffff8f6234918e00) 00000100:00000040:0.0:1775775822.140752:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918e00 x1862022710368384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775847 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140756:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775822.140758:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775822.140759:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775775822.140761:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710368384, offset 0 00000400:00000200:0.0:1775775822.140763:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775822.140765:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775822.140767:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775822.140769:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775822.140770:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775822.140772:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775822.140773:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775822.140775:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453430:453430:256:4294967295] 192.168.201.115@tcp LPNI seq info [453355:453355:8:4294967295] 00000400:00000200:0.0:1775775822.140778:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775822.140782:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775822.140785:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775822.140787:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775775822.140789:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775822.140791:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775775822.140793:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775822.140795:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775775822.140798:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775822.140799:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775775822.140800:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775822.140802:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775822.140804:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775775822.140806:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775775822.140807:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775822.140809:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918e00 x1862022710368384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775847 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140814:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775822.140816:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918e00 x1862022710368384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775847 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140819:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775822.140820:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775822.140822:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775822.140824:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775822.140825:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775822.140827:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307377164 00000100:00000001:0.0:1775775822.140828:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307380999 00000100:00000001:0.0:1775775822.140829:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307377164 00000100:00000001:0.0:1775775822.140831:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307380999 00000100:00000001:0.0:1775775822.140832:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775822.140834:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775822.140835:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775822.140837:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775822/real 1775775822] req@ffff8f6234918e00 x1862022710368384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775822.140841:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775822.140842:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775822.140843:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775775822.140844:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775775822.140845:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775775822.140846:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775822.140848:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918e00 x1862022710368384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775822.140853:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918e00 x1862022710368384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140857:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775822.140858:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775822.140859:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775822.140860:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775822.140862:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918e00 x1862022710368384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140866:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775822.140868:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918e00 x1862022710368384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140872:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775822.140874:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775822.140875:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775822.140875:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775822.140876:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775822.140878:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775822.140879:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775822.140881:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775822.140882:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775822.140884:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918e00 x1862022710368384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775822.140889:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710368384:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775822.140891:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775822.140892:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775822.140894:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918380 x1862022710368256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140898:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775822.140899:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775822.140900:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775822.140902:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775822.140903:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775822.140904:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775822.140905:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775822.140906:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:0.0:1775775822.140908:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775822.140909:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775822.140910:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918380. 00000100:00000001:0.0:1775775822.140912:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775822.140913:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775822.140914:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775822.140915:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918e00 x1862022710368384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775847 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775822.140920:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775822.140920:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775822.140921:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775822.140923:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775822.140924:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775822.140924:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775822.140926:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775822.140926:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4c400. 02000000:00000001:0.0:1775775822.140928:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775822.140928:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775822.140929:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918e00. 00000100:00000001:0.0:1775775822.140930:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775822.140931:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775822.140932:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775822.140934:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775822.140935:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775822.140936:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775822.140937:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775822.267323:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775822.267330:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775822.267335:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775822.267336:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775822.267340:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775822.267341:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775822.267405:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775822.267409:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775822.267412:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775823.291212:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775823.291221:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775823.291224:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775823.291227:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775823.291243:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775823.291247:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775823.291249:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775823.291250:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775823.291254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775823.291256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775823.291259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775823.291260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775823.291314:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775823.291317:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775823.291319:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775824.315260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775824.315274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775824.315279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775824.315281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775824.315284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775824.315285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775824.315350:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775824.315355:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775824.315359:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775825.339194:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775825.339233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775825.339241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775825.339243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775825.339247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775825.339248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775826.043292:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307380999, at 4307381056 00000100:00000010:0.0:1775775826.043305:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00010000:00000001:3.0:1775775826.363250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775826.363257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775826.363262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775826.363264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775826.363267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775826.363268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775826.363288:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775826.363290:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775826.363292:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775827.259257:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775827.259265:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775827.259268:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775827.259272:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775827.259277:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775827.259283:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12709 00000400:00000200:3.0:1775775827.259293:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775827.259299:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775827.259303:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775827.259306:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775827.259309:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775827.259311:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775827.259314:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775827.259318:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775827.259321:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775827.259323:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775827.259326:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775827.259328:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775827.259331:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775827.259334:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775827.259336:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775827.259338:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775827.259340:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775827.259343:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775827.259346:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775827.259350:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775827.259354:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775827.259357:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775827.259361:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775827.259363:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775827.259365:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775827.259374:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775775827.259377:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775827.259382:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775827.259383:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775827.259386:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775827.259389:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775827.259393:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775827.259401:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000001:3.0:1775775827.259407:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775827.259431:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710368512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775827.259443:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775827.259447:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775827.259453:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710368512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775827.259477:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775827.259481:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775827.259485:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775827.259487:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775827.259488:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775827.259491:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775827.259494:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12709 00000400:00000200:3.0:1775775827.259499:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775827.259503:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775827.259506:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775827.259510:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775827.259513:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775827.259516:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775827.259517:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775775827.259519:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:0.0:1775775827.259521:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775827.259524:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1775775827.259524:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775827.259528:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775827.259532:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000040:0.0:1775775827.259533:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71500 x1862022710368512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775775827.259535:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775827.259542:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775775827.259546:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1775775827.259547:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775775827.259548:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775827.259554:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775827.259557:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1775775827.259557:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 02000000:00000001:0.0:1775775827.259562:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775827.259563:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:0.0:1775775827.259567:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710368512:192.168.201.115@tcp:250:kworker.0 00000100:00000040:3.0:1775775827.259568:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775827.259574:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:0.0:1775775827.259575:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775827.259578:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:3.0:1775775827.259579:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:0.0:1775775827.259581:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775827.259584:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1775775827.259587:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000010:0.0:1775775827.259590:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 00000100:00000001:3.0:1775775827.259591:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775827.259594:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775827.259596:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775827.259603:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775775827.259604:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775775827.259608:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775775827.259611:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000100:00000001:3.0:1775775827.259611:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775827.259616:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000010:0.0:1775775827.259620:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000008:00000001:3.0:1775775827.259621:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775827.259625:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775775827.259630:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710368512, portal 25 00000100:00000010:3.0:1775775827.259631:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000100:00000001:0.0:1775775827.259636:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000040:3.0:1775775827.259637:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000001:0.0:1775775827.259640:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849009408 : -123823860542208 : ffff8f6202c71500) 02000000:00000001:3.0:1775775827.259642:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775827.259645:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775827.259648:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:0.0:1775775827.259650:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71500 x1862022710368512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775775827.259651:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775827.259655:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775827.259663:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:0.0:1775775827.259665:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775775827.259669:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775827.259671:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775827.259677:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00100000:3.0:1775775827.259679:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710368640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1775775827.259681:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710368512, offset 0 00000100:00000001:3.0:1775775827.259692:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1775775827.259695:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775775827.259696:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775827.259701:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710368640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775775827.259710:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775775827.259711:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775827.259715:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775827.259717:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775827.259719:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1775775827.259721:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00080000:3.0:1775775827.259722:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775827.259728:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775827.259731:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12709 00000400:00000200:0.0:1775775827.259732:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775775827.259736:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775827.259739:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:3.0:1775775827.259742:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775827.259744:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775827.259749:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:0.0:1775775827.259750:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1775775827.259752:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775827.259755:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:0.0:1775775827.259758:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:3.0:1775775827.259759:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775827.259762:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775827.259764:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775827.259767:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:0.0:1775775827.259768:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453431:453431:256:4294967295] 192.168.201.115@tcp LPNI seq info [453356:453356:8:4294967295] 00000020:00000040:3.0:1775775827.259769:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775827.259773:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775827.259778:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775827.259783:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:0.0:1775775827.259784:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775775827.259785:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775827.259789:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775827.259791:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775827.259793:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775827.259797:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775827.259801:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775827.259803:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1775775827.259807:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 02000000:00000001:3.0:1775775827.259807:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775827.259811:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775827.259814:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775827.259817:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775827.259819:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775775827.259820:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775775827.259822:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775827.259827:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000800:00000010:0.0:1775775827.259828:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000020:00000040:3.0:1775775827.259829:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775827.259834:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775827.259835:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775827.259836:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775827.259838:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:0.0:1775775827.259839:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:3.0:1775775827.259841:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775775827.259844:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 02000000:00000010:3.0:1775775827.259846:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:3.0:1775775827.259850:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775775827.259853:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00100000:3.0:1775775827.259858:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710368768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.1:1775775827.259859:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775775827.259869:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775827.259873:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000100:00000001:3.0:1775775827.259873:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775827.259877:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775827.259880:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:3.0:1775775827.259880:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710368768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775775827.259889:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1775775827.259892:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775827.259895:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000100:00000040:3.0:1775775827.259895:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:0.0:1775775827.259898:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000040:3.0:1775775827.259899:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775827.259902:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775775827.259903:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1775775827.259907:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000200:0.0:1775775827.259911:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71500 x1862022710368512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775775827.259912:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12719) 00000100:00000001:0.0:1775775827.259921:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775827.259924:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71500 x1862022710368512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775852 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.259929:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775827.259930:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775827.259934:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775827.259937:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775827.259939:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307386118 00000100:00000010:0.0:1775775827.259944:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775775827.259949:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775775827.259955:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307386118 00000100:00000010:0.0:1775775827.259957:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3600. 00000100:00000001:0.0:1775775827.259959:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775775827.259961:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775827.259963:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775827.259965:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775827.259968:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775827/real 1775775827] req@ffff8f6202c71500 x1862022710368512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775827.259974:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775827.259975:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775827.259977:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775775827.259979:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775775827.259980:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775775827.259981:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775827.259984:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71500 x1862022710368512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775827.259989:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71500 x1862022710368512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.259994:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775827.259995:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775827.259996:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775827.259997:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775827.260000:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71500 x1862022710368512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.260004:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775827.260007:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71500 x1862022710368512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.260011:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775827.260013:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775827.260014:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775827.260014:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775827.260016:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775827.260018:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775827.260020:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775827.260023:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775827.260024:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775827.260027:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71500 x1862022710368512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775827.260032:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710368512:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775775827.260035:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775827.260036:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775827.260038:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71500 x1862022710368512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.260042:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775827.260043:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775827.260045:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775827.260047:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775827.260048:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775827.260049:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775775827.260051:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775827.260053:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35800. 02000000:00000001:0.0:1775775827.260055:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775827.260057:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775827.260058:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71500. 00000100:00000001:0.0:1775775827.260060:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775827.260061:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775827.260062:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775827.260068:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775827.260068:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775827.260070:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775827.260071:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775827.260082:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775827.260084:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73480 x1862022710368640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.260093:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775827.260094:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775827.260096:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775827.260097:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775827.260098:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775827.260100:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710368640:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775827.260102:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775827.260103:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775827.260104:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775827.260106:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775827.260108:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775827.260109:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775827.260112:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775827.260114:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775775827.260116:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775775827.260118:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710368640, portal 10 00000100:00000001:0.0:1775775827.260119:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775827.260120:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849017472 : -123823860534144 : ffff8f6202c73480) 00000100:00000040:0.0:1775775827.260124:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73480 x1862022710368640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.260129:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775827.260130:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775827.260131:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775775827.260182:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710368640, offset 0 00000400:00000200:0.0:1775775827.260186:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775827.260189:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775827.260192:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775827.260194:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775827.260195:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775827.260197:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775827.260199:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775827.260202:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453432:453432:256:4294967295] 192.168.201.115@tcp LPNI seq info [453357:453357:8:4294967295] 00000400:00000200:0.0:1775775827.260204:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775827.260210:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775827.260215:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775827.260219:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775775827.260223:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775827.260226:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775775827.260230:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775827.260234:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775775827.260237:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775827.260239:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775775827.260242:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775827.260246:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775827.260249:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775775827.260251:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775775827.260254:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775827.260258:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73480 x1862022710368640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.260266:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775827.260269:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73480 x1862022710368640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775852 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.260287:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775827.260292:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775827.260295:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775827.260298:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775775827.260301:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775827.260302:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307382284 00000100:00000001:0.0:1775775827.260305:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307386118 00000100:00000001:0.0:1775775827.260306:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307382284 00000100:00000001:0.0:1775775827.260308:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307386118 00000100:00000001:0.0:1775775827.260309:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775827.260311:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775827.260313:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775827.260316:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775827/real 1775775827] req@ffff8f6202c73480 x1862022710368640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775827.260321:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775827.260322:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775827.260323:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775775827.260325:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775775827.260326:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775775827.260327:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775827.260329:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73480 x1862022710368640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775827.260335:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73480 x1862022710368640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.260339:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775827.260340:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775827.260341:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775827.260342:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775827.260344:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73480 x1862022710368640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.260348:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775827.260350:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73480 x1862022710368640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.260354:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775827.260355:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775827.260356:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775827.260357:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775827.260358:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775827.260361:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775827.260363:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775827.260366:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775827.260367:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775827.260370:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73480 x1862022710368640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775827.260374:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710368640:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775827.260377:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775775827.260379:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71c00 x1862022710368768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.260382:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775827.260383:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775827.260385:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775775827.260386:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775827.260387:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775827.260389:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710368768:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775827.260391:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775775827.260392:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775775827.260393:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775827.260394:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775775827.260397:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775827.260398:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775775827.260400:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775775827.260402:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775775827.260403:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775775827.260405:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710368768, portal 10 00000100:00000001:0.0:1775775827.260407:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775775827.260408:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849011200 : -123823860540416 : ffff8f6202c71c00) 00000100:00000040:0.0:1775775827.260411:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71c00 x1862022710368768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.260415:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775775827.260417:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775775827.260419:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775775827.260420:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710368768, offset 0 00000400:00000200:0.0:1775775827.260423:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775775827.260425:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775827.260428:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775775827.260430:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775775827.260432:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775775827.260433:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775775827.260435:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775775827.260437:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453433:453433:256:4294967295] 192.168.201.115@tcp LPNI seq info [453358:453358:8:4294967295] 00000400:00000200:0.0:1775775827.260440:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775827.260444:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775775827.260447:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775827.260449:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775775827.260451:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775775827.260453:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775775827.260456:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775775827.260457:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775775827.260460:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775775827.260461:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775775827.260463:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775775827.260466:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775775827.260467:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775775827.260469:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775775827.260470:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775827.260472:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71c00 x1862022710368768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.260477:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775827.260479:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71c00 x1862022710368768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775852 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.260483:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775827.260485:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775775827.260486:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775827.260488:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775775827.260490:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775775827.260491:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307382284 00000100:00000001:0.0:1775775827.260493:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307386118 00000100:00000001:0.0:1775775827.260494:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307382284 00000100:00000001:0.0:1775775827.260495:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307386118 00000100:00000001:0.0:1775775827.260496:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775775827.260498:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775827.260499:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775775827.260501:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775775827/real 1775775827] req@ffff8f6202c71c00 x1862022710368768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775775827.260505:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775775827.260506:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775775827.260507:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775775827.260508:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775775827.260509:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775775827.260510:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775775827.260512:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71c00 x1862022710368768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775775827.260517:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71c00 x1862022710368768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.260521:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775775827.260522:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775827.260523:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775827.260524:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775775827.260526:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71c00 x1862022710368768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.260530:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775775827.260532:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71c00 x1862022710368768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.260537:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775775827.260538:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775827.260539:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775775827.260540:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775827.260541:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775775827.260542:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775775827.260544:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775775827.260545:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775775827.260547:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775775827.260549:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71c00 x1862022710368768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775775827.260553:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710368768:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775775827.260555:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775827.260556:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775827.260558:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73480 x1862022710368640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.260562:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775827.260563:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775827.260564:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775775827.260565:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775827.260566:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775827.260567:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775827.260568:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775827.260569:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37400. 02000000:00000001:0.0:1775775827.260571:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775827.260572:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775827.260573:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73480. 00000100:00000001:0.0:1775775827.260575:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775827.260576:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775827.260577:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775827.260579:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71c00 x1862022710368768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775775852 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775827.260583:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775775827.260584:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775775827.260585:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775775827.260586:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775775827.260587:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775775827.260587:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775775827.260588:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775775827.260589:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35400. 02000000:00000001:0.0:1775775827.260590:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775775827.260591:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775775827.260592:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71c00. 00000100:00000001:0.0:1775775827.260593:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775775827.260594:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775827.260595:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775827.260597:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775775827.260598:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775827.260599:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775775827.260600:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775827.387278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775827.387284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775827.387290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775827.387292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775827.387296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775827.387297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775827.387363:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775827.387366:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775827.387369:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775828.411246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775828.411255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775828.411263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775828.411267:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775828.411270:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775828.411274:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775828.411278:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775828.411281:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775828.411283:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775828.411286:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775828.411289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775828.411290:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775828.411291:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775828.411292:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775828.411294:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775828.411295:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775828.411296:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775828.411297:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775828.411391:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775828.411397:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775828.411401:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775829.435288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775829.435295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775829.435300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775829.435301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775829.435304:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775829.435305:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775829.435367:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775829.435371:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775829.435374:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775830.459243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775830.459252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775830.459260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775830.459261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775830.459267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775830.459268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775831.163233:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307386118, at 4307386176 00000100:00000010:0.0:1775775831.163254:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00010000:00000001:3.0:1775775831.483269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775831.483276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775831.483281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775831.483282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775831.483285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775831.483286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775831.483314:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775831.483317:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775831.483321:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775832.379267:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775832.379280:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775832.379284:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775832.379290:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775832.379298:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775832.379310:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12714 00000400:00000200:0.0:1775775832.379320:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775832.379328:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775832.379331:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775832.379335:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775832.379339:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775832.379341:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775832.379343:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775832.379347:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775832.379348:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775832.379351:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775832.379354:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775832.379355:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775832.379357:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775832.379360:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775832.379361:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775832.379363:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775832.379364:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775832.379366:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775832.379369:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775832.379370:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775832.379374:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775832.379376:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775832.379379:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775832.379381:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775832.379383:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775832.379393:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063800. 00000020:00000040:0.0:1775775832.379396:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775832.379400:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775832.379401:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775832.379404:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775832.379414:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775832.379419:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775832.379428:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6000. 00000100:00000001:0.0:1775775832.379434:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775832.379443:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063800 x1862022710368896/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775832.379457:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775832.379462:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775832.379466:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000015d81d25] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063800 x1862022710368896/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775832.379488:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775832.379491:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775832.379494:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775832.379495:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775832.379497:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775832.379499:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775832.379501:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12714 00000400:00000200:0.0:1775775832.379504:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775832.379506:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775832.379508:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775832.379510:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775832.379512:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775832.379514:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775832.379516:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775832.379518:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775832.379519:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775832.379520:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775832.379522:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775832.379525:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775832.379527:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775832.379532:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775832.379534:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775832.379538:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775832.379543:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775832.379548:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775832.379559:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775832.379563:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775832.379566:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775832.379572:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775832.379577:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775832.379580:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775832.379585:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775832.379588:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775832.379591:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775832.379595:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060380. 00000020:00000040:0.0:1775775832.379596:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775832.379598:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775832.379600:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775832.379603:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775832.379605:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775832.379607:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775832.379612:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000001:0.0:1775775832.379617:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775832.379624:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060380 x1862022710369024/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775832.379633:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775832.379635:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775832.379638:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000614ace82] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060380 x1862022710369024/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775832.379651:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775832.379655:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775832.379665:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775832.379669:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775832.379672:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775832.379678:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775832.379685:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12714 00000400:00000200:0.0:1775775832.379691:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775832.379697:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775832.379702:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775832.379712:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775832.379717:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775832.379720:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775832.379724:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775832.379729:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775832.379732:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775832.379734:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775832.379737:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775832.379742:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775832.379746:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775832.379755:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775832.379759:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775832.379764:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775832.379769:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775832.379773:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775832.379776:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775832.379779:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775832.379784:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775832.379789:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775832.379794:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775832.379799:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775832.379809:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775832.379815:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775832.379819:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775832.379824:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000010:0.0:1775775832.379826:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062680. 00000800:00000200:1.1:1775775832.379827:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 11 secs later) 00000020:00000040:0.0:1775775832.379831:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775832.379838:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775832.379841:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775832.379843:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775832.379846:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775832.379848:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775832.379852:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4400. 00000100:00000001:0.0:1775775832.379857:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775832.379865:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062680 x1862022710369152/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775832.379871:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775832.379875:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775832.379878:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000082c7985e] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062680 x1862022710369152/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775832.379885:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775832.379887:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775832.379890:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775832.379895:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775832.379902:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775832.379909:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12725) 00000800:00000200:1.1:1775775832.380239:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775832.380241:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1775775832.380547:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775832.380549:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 11 secs later) 00010000:00000001:3.0:1775775832.507298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775832.507307:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775832.507316:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775832.507317:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775832.507321:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775832.507322:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775832.507386:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775832.507392:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775832.507396:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775833.531292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775833.531301:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775833.531303:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775833.531307:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775833.531312:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775833.531315:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775833.531317:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775833.531318:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775833.531322:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775833.531323:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775833.531326:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775833.531326:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775833.531394:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775833.531399:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775833.531402:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775834.555271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775834.555277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775834.555282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775834.555282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775834.555286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775834.555287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775834.555368:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775834.555374:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775834.555377:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775835.579246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775835.579250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775835.579254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775835.579256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775835.579259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775835.579259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775836.283242:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307391238, at 4307391296 00000100:00000010:0.0:1775775836.283256:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2b40. 00000800:00000200:2.0:1775775836.283296:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775775836.283307:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775775836.283311:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1775775836.283464:0:2405:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:3.0:1775775836.603252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775836.603257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775836.603261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775836.603262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775836.603265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775836.603266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775836.603285:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775836.603287:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775836.603289:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775837.499244:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775837.499253:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775837.499257:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775837.499261:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775837.499267:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775837.499277:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12720 00000400:00000200:0.0:1775775837.499291:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775837.499298:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775837.499303:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775837.499309:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775837.499313:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775837.499316:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775837.499319:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775837.499324:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775837.499326:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775837.499329:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775837.499331:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775837.499334:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775837.499336:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775837.499339:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775837.499341:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775837.499344:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775837.499346:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775837.499349:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775837.499353:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775837.499356:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775837.499359:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775837.499362:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775837.499365:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775837.499367:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775837.499370:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775837.499380:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062d80. 00000020:00000040:0.0:1775775837.499384:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775837.499388:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775837.499390:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775837.499393:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775837.499396:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775837.499400:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775837.499408:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6c00. 00000100:00000001:0.0:1775775837.499415:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775837.499425:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062d80 x1862022710369280/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775837.499437:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775837.499441:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775837.499446:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009965f80b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062d80 x1862022710369280/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775837.499469:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775837.499472:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775837.499476:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775837.499477:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775837.499479:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775837.499482:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775837.499485:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12720 00000400:00000200:0.0:1775775837.499490:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775837.499494:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775837.499497:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775837.499500:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775837.499503:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775837.499506:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775837.499509:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775837.499513:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775837.499516:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775837.499521:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775837.499523:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775837.499528:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775837.499534:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775837.499545:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775837.499551:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775837.499556:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775837.499573:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775837.499579:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775837.499586:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775837.499591:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775837.499596:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775837.499604:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775837.499607:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775837.499611:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775837.499616:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775837.499622:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775837.499624:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775837.499632:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062a00. 00000020:00000040:0.0:1775775837.499638:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775837.499644:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775837.499647:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775837.499651:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775837.499655:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775837.499659:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775837.499666:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5800. 00000100:00000001:0.0:1775775837.499673:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775837.499683:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062a00 x1862022710369408/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775837.499703:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775837.499709:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775837.499718:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009753f9f6] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062a00 x1862022710369408/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775837.499743:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775837.499747:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775837.499755:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775837.499759:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775837.499762:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775837.499769:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775837.499776:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12720 00000400:00000200:0.0:1775775837.499782:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775837.499789:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775837.499793:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775837.499800:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775837.499805:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775837.499811:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:1.1:1775775837.499811:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775775837.499818:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000200:1.1:1775775837.499819:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000020:00000001:0.0:1775775837.499823:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775837.499826:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775837.499830:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775837.499834:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775837.499838:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775837.499842:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775837.499847:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775837.499851:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775837.499857:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775837.499863:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775837.499867:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775837.499873:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775837.499879:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775837.499881:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775837.499887:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775837.499892:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775837.499896:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775837.499898:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775837.499901:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775837.499903:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775837.499910:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061500. 00000020:00000040:0.0:1775775837.499916:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775837.499923:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775837.499926:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775837.499931:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775837.499935:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775837.499938:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775837.499945:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7000. 00000100:00000001:0.0:1775775837.499951:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775837.499959:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061500 x1862022710369536/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775837.499970:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775837.499974:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775837.499981:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb8d992] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061500 x1862022710369536/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775837.500007:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775837.500010:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775837.500016:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775837.500023:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775837.500031:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775837.500040:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12730) 00000800:00000200:1.1:1775775837.500323:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775837.500326:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775775837.500572:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775837.500574:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00010000:00000001:3.0:1775775837.627256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775837.627261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775837.627266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775837.627267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775837.627270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775837.627271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775837.627340:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775837.627344:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775837.627348:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775838.651261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775838.651267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775838.651273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775838.651277:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775838.651279:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775838.651282:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775838.651285:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775838.651288:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775838.651290:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775838.651293:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775838.651297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775838.651298:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775838.651300:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775838.651301:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775838.651303:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775838.651305:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775838.651307:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775838.651309:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775838.651378:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775838.651382:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775838.651385:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775839.675229:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775839.675234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775839.675239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775839.675241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775839.675244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775839.675246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775839.675333:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775839.675371:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775839.675374:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775840.699241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775840.699248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775840.699255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775840.699256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775840.699260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775840.699262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775841.403239:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307396358, at 4307396416 00000100:00000010:0.0:1775775841.403255:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e20c0. 00010000:00000001:3.0:1775775841.723231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775841.723236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775841.723240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775841.723241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775841.723244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775841.723245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775841.723263:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775841.723266:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775841.723268:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775842.619236:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775842.619283:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775842.619286:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775842.619290:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775842.619294:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775842.619302:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12725 00000400:00000200:2.0:1775775842.619311:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775842.619317:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775842.619321:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775842.619325:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775842.619328:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775842.619330:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775842.619333:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775842.619337:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775842.619339:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775842.619342:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775842.619359:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775842.619362:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775842.619365:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775842.619368:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775842.619370:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775842.619372:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775842.619373:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775842.619376:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775842.619378:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775842.619380:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775842.619382:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775842.619385:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775842.619387:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775842.619389:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775842.619391:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775842.619398:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775775842.619401:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775842.619404:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775842.619406:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775842.619408:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775842.619410:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775842.619413:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775842.619420:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775775842.619424:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775842.619432:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710369664/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775842.619440:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775842.619444:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775842.619447:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710369664/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775842.619463:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775842.619466:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775842.619468:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775842.619469:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775842.619470:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775842.619473:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775842.619475:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12725 00000400:00000200:2.0:1775775842.619479:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775842.619481:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775842.619483:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775842.619485:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775842.619487:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775842.619489:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775842.619491:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775842.619495:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775842.619497:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775842.619499:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775842.619503:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775842.619504:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775842.619507:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775842.619512:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775842.619516:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775842.619520:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775842.619524:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775842.619527:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775842.619532:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775842.619536:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775842.619539:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775842.619544:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775842.619549:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775842.619551:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775842.619556:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775842.619560:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775842.619562:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775842.619566:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775775842.619568:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775842.619573:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775842.619575:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775842.619577:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775842.619578:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775842.619581:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775842.619587:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775775842.619591:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775842.619597:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710369792/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775842.619604:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775842.619608:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775842.619614:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710369792/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775842.619625:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775842.619630:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775842.619635:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775842.619639:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775842.619642:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775842.619646:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775842.619651:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12725 00000400:00000200:2.0:1775775842.619656:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775842.619660:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775842.619664:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775842.619668:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775842.619673:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775842.619678:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775842.619680:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775842.619681:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775842.619682:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775842.619683:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775842.619685:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775842.619689:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775842.619691:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775842.619696:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775842.619701:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775842.619703:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775842.619707:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775842.619709:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775842.619714:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775842.619719:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775842.619723:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775842.619726:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775842.619729:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775842.619734:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775842.619737:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775842.619739:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775842.619741:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775842.619744:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775775842.619748:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775842.619753:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775842.619755:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775842.619759:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775842.619762:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775842.619764:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775842.619768:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775775842.619772:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775842.619779:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710369920/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775842.619790:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.1:1775775842.619791:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:2.0:1775775842.619794:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775842.619796:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000100:00000040:2.0:1775775842.619799:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710369920/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775842.619815:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775842.619818:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775842.619824:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775842.619829:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775842.619833:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775842.619838:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12735) 00000800:00000200:1.1:1775775842.620122:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775842.620124:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775775842.620359:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775842.620361:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00010000:00000001:3.0:1775775842.747215:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775842.747221:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775842.747227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775842.747228:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775842.747231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775842.747231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775842.747313:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775842.747320:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775842.747325:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775843.771275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775843.771285:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775843.771287:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775843.771291:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775843.771298:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775843.771301:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775843.771303:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775843.771306:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775843.771310:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775843.771311:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775843.771315:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775843.771317:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775843.771375:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775843.771379:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775843.771382:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775844.795284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775844.795295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775844.795303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775844.795305:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775844.795309:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775844.795311:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775844.795371:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775844.795376:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775844.795379:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775845.819211:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775845.819216:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775845.819220:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775845.819221:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775845.819224:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775845.819225:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775846.523278:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307401478, at 4307401536 00000100:00000010:0.0:1775775846.523304:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ba0. 00010000:00000001:3.0:1775775846.843268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775846.843273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775846.843277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775846.843278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775846.843281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775846.843282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775846.843303:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775846.843305:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775846.843307:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775847.739258:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775847.739267:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775847.739272:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775847.739278:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775847.739295:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775847.739304:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12730 00000400:00000200:3.0:1775775847.739313:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775847.739319:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775847.739323:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775847.739326:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775847.739328:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775847.739330:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775847.739332:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775847.739335:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775847.739338:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775847.739340:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775847.739342:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775847.739343:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775847.739345:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775847.739348:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775847.739350:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775847.739352:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775847.739354:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775847.739357:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775847.739360:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775847.739362:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775847.739366:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775847.739370:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775847.739373:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775847.739375:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775847.739378:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775847.739387:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775775847.739392:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775847.739396:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775847.739398:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775847.739401:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775847.739403:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775847.739418:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775847.739428:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000001:3.0:1775775847.739435:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775847.739445:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710370048/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775847.739456:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775847.739460:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775847.739463:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710370048/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775847.739487:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775847.739489:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775847.739492:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775847.739493:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775847.739494:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775847.739496:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775847.739498:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12730 00000400:00000200:3.0:1775775847.739501:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775847.739503:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775847.739505:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775847.739509:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775847.739511:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775847.739513:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775847.739515:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775847.739516:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775847.739518:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775847.739521:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775847.739523:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775847.739527:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775847.739530:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775847.739535:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775847.739537:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775847.739541:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775847.739545:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775847.739549:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775847.739562:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775847.739566:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775847.739568:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775847.739574:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775847.739578:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775847.739582:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775847.739586:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775847.739591:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775847.739597:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775847.739601:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000020:00000040:3.0:1775775847.739603:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775847.739605:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775847.739608:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775847.739611:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775847.739612:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775847.739615:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775847.739620:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000001:3.0:1775775847.739624:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775847.739631:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710370176/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775847.739639:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775847.739642:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775847.739644:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710370176/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775847.739657:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775847.739661:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775847.739665:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775847.739667:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775847.739669:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775847.739674:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775847.739679:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12730 00000400:00000200:3.0:1775775847.739687:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775847.739690:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775847.739694:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775847.739701:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775847.739703:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775847.739707:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775847.739709:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775847.739712:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775847.739713:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775847.739714:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775847.739717:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775847.739720:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775847.739722:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775847.739725:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775847.739729:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775847.739733:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775847.739735:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775847.739737:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775847.739743:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775847.739745:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775847.739748:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775847.739750:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775847.739755:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775847.739758:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775847.739761:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775847.739765:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775847.739767:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775847.739771:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775775847.739776:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775847.739779:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775847.739781:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775847.739784:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775847.739785:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775847.739787:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775847.739792:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000100:00000001:3.0:1775775847.739796:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775847.739820:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710370304/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775847.739832:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775847.739834:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775847.739841:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710370304/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1775775847.739846:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775775847.739855:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775847.739857:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775847.739860:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775847.739862:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775847.739865:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775847.739869:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12740) 00000800:00000200:1.1:1775775847.740027:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775847.740209:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00010000:00000001:3.0:1775775847.867213:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775847.867218:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775847.867222:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775847.867223:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775847.867226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775847.867227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775847.867290:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775847.867293:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775847.867296:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775848.891293:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775848.891300:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775848.891306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775848.891322:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775848.891324:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775848.891328:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775848.891331:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775848.891333:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775848.891336:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775848.891339:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775848.891343:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775848.891344:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775848.891347:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775848.891348:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775848.891350:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775848.891351:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775848.891353:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775848.891355:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775848.891432:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775848.891435:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775848.891438:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775849.915282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775849.915292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775849.915300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775849.915302:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775849.915306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775849.915308:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775849.915373:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775849.915377:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775849.915380:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775775850.811416:0:2400:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:2.0:1775775850.811450:0:2400:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1775775850.811459:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775850.811462:0:2400:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:2.0:1775775850.811466:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775850.811469:0:2400:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:2.0:1775775850.811473:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775850.811476:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:2.0:1775775850.811480:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775850.811484:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775850.811489:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3990 00000400:00000010:2.0:1775775850.811492:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3990. 00000100:00000001:2.0:1775775850.811497:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775850.811505:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c70380 x1862022710370048/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775872 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775850.811527:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775850.811568:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70380 x1862022710370048/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775872 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775850.811574:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775850.811576:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775775850.811586:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775850.811588:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230400. 00000400:00000200:2.0:1775775850.811591:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775850.811596:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775850.811600:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3e58 00000400:00000010:2.0:1775775850.811603:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3e58. 00000100:00000001:2.0:1775775850.811607:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775850.811614:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c70e00 x1862022710370176/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775872 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775850.811628:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775850.811633:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70e00 x1862022710370176/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775872 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775850.811646:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775850.811649:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775775850.811658:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775850.811663:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230600. 00000400:00000200:2.0:1775775850.811669:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775850.811679:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775850.811684:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3908 00000400:00000010:2.0:1775775850.811687:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3908. 00000100:00000001:2.0:1775775850.811692:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775850.811699:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c70000 x1862022710370304/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775872 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775850.811714:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775850.811720:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70000 x1862022710370304/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775872 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775850.811747:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775850.811766:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:3.0:1775775850.939309:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775850.939319:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775850.939328:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775850.939330:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775850.939341:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775850.939342:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775851.643274:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307406598, at 4307406656 00000100:00000010:0.0:1775775851.643289:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e27e0. 00010000:00000001:3.0:1775775851.963253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775851.963259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775851.963266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775851.963267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775851.963272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775851.963273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775851.963303:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775851.963306:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775851.963308:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775852.859263:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775852.859274:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775852.859276:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775852.859280:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775852.859285:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775852.859292:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12735 00000400:00000200:2.0:1775775852.859302:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775852.859309:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775852.859313:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775852.859318:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775852.859321:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775852.859326:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775852.859328:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775852.859331:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775852.859334:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775852.859337:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775852.859341:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775852.859343:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775852.859346:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775852.859351:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775852.859353:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775852.859355:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775852.859356:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775852.859358:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775852.859361:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775852.859363:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775852.859366:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775852.859370:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775852.859373:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775852.859375:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775852.859378:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775852.859394:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775775852.859398:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775852.859401:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775852.859403:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775852.859406:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775852.859409:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775852.859413:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775852.859419:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775852.859426:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775852.859437:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710370432/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775852.859447:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775852.859452:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775852.859457:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710370432/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775852.859484:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775852.859488:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775852.859491:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775852.859492:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775852.859494:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775852.859497:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775852.859500:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12735 00000400:00000200:2.0:1775775852.859504:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775852.859507:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775852.859510:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775852.859513:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775852.859515:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775852.859517:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775852.859520:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775852.859522:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775852.859524:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775852.859526:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775852.859528:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775852.859530:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775852.859532:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775852.859535:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775852.859537:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775852.859539:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775852.859541:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775852.859543:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775852.859545:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775852.859548:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775852.859549:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775852.859557:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775852.859560:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775852.859561:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775852.859566:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775852.859569:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775852.859571:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775852.859575:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775775852.859577:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775852.859580:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775852.859581:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775852.859583:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775852.859584:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775852.859586:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775852.859595:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775852.859599:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775852.859604:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710370560/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775852.859612:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775852.859614:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775852.859619:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710370560/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775852.859627:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775852.859629:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775852.859633:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775852.859634:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775852.859635:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775852.859637:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775852.859639:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12735 00000400:00000200:2.0:1775775852.859643:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775852.859646:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775852.859648:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775852.859651:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775852.859653:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775852.859655:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775852.859657:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775852.859659:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775852.859660:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775852.859662:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775852.859664:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775852.859666:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775852.859668:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775852.859670:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775852.859672:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775852.859674:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775852.859676:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775852.859678:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775852.859681:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775852.859684:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775852.859686:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775852.859690:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775852.859693:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775852.859694:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775852.859697:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775852.859700:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775852.859702:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775852.859706:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775775852.859708:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775852.859711:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775852.859712:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775852.859714:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775852.859731:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775852.859733:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775852.859737:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775775852.859740:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775852.859745:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710370688/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775852.859753:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775852.859756:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775852.859760:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710370688/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775852.859768:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775852.859771:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775852.859777:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775852.859780:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775852.859783:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775852.859789:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12745) 00000800:00000200:1.1:1775775852.860044:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775852.860090:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775775852.860466:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775852.860469:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775775852.860763:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775852.860766:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00010000:00000001:3.0:1775775852.987248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775852.987252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775852.987256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775852.987257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775852.987260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775852.987261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775852.987322:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775852.987326:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775852.987330:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775854.011234:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775854.011241:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775854.011244:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775854.011247:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775854.011252:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775854.011255:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775854.011257:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775854.011259:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775854.011263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775854.011265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775854.011269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775854.011270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775854.011390:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775854.011393:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775854.011396:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775855.035235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775855.035241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775855.035245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775855.035246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775855.035250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775855.035251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775855.035343:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775855.035348:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775855.035352:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775856.059228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775856.059234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775856.059238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775856.059238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775856.059241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775856.059242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775856.763312:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307411719, at 4307411776 00000100:00000010:0.0:1775775856.763348:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2e40. 00010000:00000001:3.0:1775775857.083277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775857.083284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775857.083289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775857.083290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775857.083293:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775857.083294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775857.083397:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775857.083400:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775857.083402:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775857.979247:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775857.979258:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775857.979262:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775857.979267:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775857.979272:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775857.979281:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12740 00000400:00000200:3.0:1775775857.979292:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775857.979298:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775857.979302:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775857.979305:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775857.979309:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775857.979311:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775857.979313:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775857.979316:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775857.979318:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775857.979320:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775857.979323:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775857.979325:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775857.979327:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775857.979329:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775857.979331:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775857.979332:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775857.979333:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775857.979336:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775857.979338:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775857.979340:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775857.979343:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775857.979345:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775857.979348:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775857.979349:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775857.979351:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775857.979360:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73100. 00000020:00000040:3.0:1775775857.979363:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775857.979366:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775857.979368:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775857.979370:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775857.979373:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775857.979377:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775857.979386:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775775857.979392:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775857.979403:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73100 x1862022710370816/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775857.979413:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775857.979418:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775857.979422:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f67e6552] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73100 x1862022710370816/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775857.979442:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775857.979444:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775857.979447:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775857.979448:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775857.979449:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775857.979451:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775857.979453:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12740 00000400:00000200:3.0:1775775857.979455:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775857.979457:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775857.979459:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775857.979462:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775857.979464:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775857.979466:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775857.979468:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775857.979470:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775857.979471:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775857.979471:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775857.979473:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775857.979475:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775857.979477:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775857.979480:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775857.979482:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775857.979485:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775857.979489:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775857.979491:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775857.979494:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775857.979496:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775857.979498:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775857.979505:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775857.979514:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775857.979519:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775857.979524:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775857.979527:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775857.979533:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775857.979541:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775775857.979545:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775857.979548:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775857.979551:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775857.979552:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775857.979554:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775857.979557:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775857.979562:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000100:00000001:3.0:1775775857.979566:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775857.979571:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710370944/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775857.979584:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775857.979588:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775857.979592:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710370944/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775857.979600:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775857.979602:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775857.979605:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775857.979608:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775857.979609:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775857.979614:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775857.979617:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12740 00000400:00000200:3.0:1775775857.979624:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775857.979643:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775857.979644:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775857.979653:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775857.979657:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775857.979661:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775857.979663:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775857.979667:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775857.979669:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775857.979672:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775857.979674:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775857.979676:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775857.979680:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775857.979684:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775857.979685:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775857.979687:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775857.979688:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775857.979689:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775857.979691:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775857.979692:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775857.979693:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775857.979696:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775857.979697:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775857.979698:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775857.979699:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775857.979701:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775857.979702:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775857.979705:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000020:00000040:3.0:1775775857.979707:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775857.979709:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775857.979710:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775857.979711:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775857.979712:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775857.979714:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775857.979717:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000100:00000001:3.0:1775775857.979719:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775857.979723:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710371072/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775857.979735:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775857.979739:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775857.979746:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710371072/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775857.979759:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775857.979763:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775857.979770:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775857.979775:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775857.979782:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775857.979789:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12750) 00000800:00000200:1.1:1775775857.979823:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775857.979826:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775775857.980193:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775857.980195:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775775857.980429:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775857.980432:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00010000:00000001:3.0:1775775858.107242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775858.107249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775858.107254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775858.107255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775858.107258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775858.107259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775858.107319:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775858.107322:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775858.107326:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775859.003263:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775859.003271:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775859.003274:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775859.003278:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775859.003282:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775859.003286:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775859.003289:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775859.003291:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775859.003294:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775859.131265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775859.131281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775859.131290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775859.131295:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775859.131298:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775859.131302:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775859.131306:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775859.131309:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775859.131311:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775859.131314:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775859.131318:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775859.131319:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775859.131320:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775859.131321:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775859.131323:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775859.131324:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775859.131325:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775859.131327:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775859.131400:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775859.131406:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775859.131410:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775860.027273:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775860.027280:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775860.027282:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775860.027288:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775860.027292:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775860.027296:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775860.027299:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775860.027301:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775860.027304:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775860.155243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775860.155252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775860.155260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775860.155262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775860.155266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775860.155267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775860.155342:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775860.155346:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775860.155349:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775861.051263:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775861.051274:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775861.051276:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775861.051284:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775861.051287:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775861.051292:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775861.051295:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775861.051297:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775861.051300:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775861.179266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775861.179274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775861.179279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775861.179280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775861.179283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775861.179284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775861.883263:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307416838, at 4307416896 00000100:00000010:0.0:1775775861.883285:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e28a0. 00000100:00000001:3.0:1775775862.075237:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775862.075243:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775862.075246:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775862.075252:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775862.075256:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775862.075259:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775862.075262:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775862.075264:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775862.075267:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775862.203235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775862.203244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775862.203263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775862.203264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775862.203268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775862.203269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775862.203292:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775862.203295:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775862.203299:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775863.099295:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775863.099306:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775863.099309:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775863.099316:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775863.099323:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775863.099333:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12745 00000400:00000200:3.0:1775775863.099345:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775863.099353:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775863.099356:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775863.099359:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775863.099368:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775863.099371:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775863.099373:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775863.099377:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775863.099380:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775863.099383:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775863.099385:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775863.099387:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775863.099390:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775863.099392:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775863.099394:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775863.099395:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775863.099397:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775863.099400:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775863.099402:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775863.099404:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775863.099407:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775863.099410:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775863.099413:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775863.099415:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775863.099417:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775863.099427:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775775863.099430:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775863.099434:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775863.099436:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775863.099439:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775863.099441:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775863.099464:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775863.099475:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000100:00000001:3.0:1775775863.099482:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775863.099492:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710371200/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775863.099505:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775863.099512:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775863.099518:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710371200/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775863.099588:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775863.099592:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775863.099596:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775863.099599:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775863.099601:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775863.099605:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775863.099609:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12745 00000400:00000200:3.0:1775775863.099616:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775863.099625:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775863.099630:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775863.099637:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775863.099641:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775863.099646:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775863.099649:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775863.099652:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775863.099654:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775863.099656:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775863.099660:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775863.099663:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775863.099668:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775863.099674:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775863.099679:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775863.099683:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775863.099685:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775863.099687:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775863.099692:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775863.099695:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775863.099697:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775863.099704:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775863.099708:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775863.099714:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775863.099718:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775863.099723:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775863.099727:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775863.099731:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775775863.099736:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775863.099738:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775863.099739:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775863.099741:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775863.099742:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775863.099744:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775863.099751:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000100:00000001:3.0:1775775863.099755:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775863.099761:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710371328/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775863.099774:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775863.099778:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775863.099787:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710371328/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775863.099803:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775863.099808:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775863.099813:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775863.099815:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775863.099818:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775863.099824:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775863.099827:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12745 00000400:00000200:3.0:1775775863.099832:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775863.099837:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775863.099841:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775863.099847:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775863.099849:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775863.099854:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775863.099856:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775863.099860:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775863.099863:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775863.099866:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775863.099869:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775863.099873:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775863.099893:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775863.099899:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775863.099903:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775863.099906:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775863.099910:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775863.099912:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775863.099914:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775863.099918:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775863.099922:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775863.099927:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775863.099931:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775863.099933:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775863.099937:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775863.099957:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775863.099959:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775863.099962:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775775863.099964:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775863.099966:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775863.099970:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775863.099972:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775863.099973:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775863.099975:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775863.099981:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775775863.099985:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775863.099994:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710371456/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1775775863.099997:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775863.100003:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000100:00000001:3.0:1775775863.100026:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775863.100031:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775863.100038:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710371456/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775863.100048:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775863.100054:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775863.100061:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775863.100064:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775863.100067:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775863.100074:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12755) 00000800:00000200:1.1:1775775863.100485:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775863.100487:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775775863.100846:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775863.100848:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00010000:00000001:3.0:1775775863.227242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775863.227251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775863.227258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775863.227260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775863.227264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775863.227266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775863.227322:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775863.227326:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775863.227329:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775864.251272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775864.251284:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775864.251288:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775864.251293:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775864.251300:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775864.251303:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775864.251306:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775864.251309:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775864.251314:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775864.251316:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775864.251319:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775864.251320:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775864.251358:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775864.251362:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775864.251365:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775865.275230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775865.275236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775865.275240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775865.275241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775865.275244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775865.275245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775865.275312:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775865.275316:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775865.275319:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775866.299245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775866.299250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775866.299254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775866.299255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775866.299258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775866.299259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775867.003247:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307421958, at 4307422016 00000100:00000010:0.0:1775775867.003261:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2a20. 00010000:00000001:3.0:1775775867.323207:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775867.323212:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775867.323216:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775867.323217:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775867.323220:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775867.323221:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775867.323238:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775867.323240:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775867.323242:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775868.219247:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775868.219263:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775868.219275:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775868.219284:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775868.219288:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775868.219298:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12750 00000400:00000200:3.0:1775775868.219310:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775868.219318:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775868.219322:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775868.219326:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775868.219330:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775868.219333:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775868.219336:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775868.219341:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775868.219343:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775868.219347:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775868.219350:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775868.219353:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775868.219356:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775868.219359:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775868.219362:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775868.219364:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775868.219366:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775868.219370:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775868.219373:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775868.219377:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775868.219380:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775868.219383:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775868.219386:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775868.219388:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775868.219390:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775868.219401:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775775868.219404:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775868.219409:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775868.219410:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775868.219413:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775868.219416:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775868.219422:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775868.219434:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000001:3.0:1775775868.219442:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775868.219473:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710371584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775868.219483:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775868.219488:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775868.219492:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710371584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775868.219514:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775868.219516:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775868.219520:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775868.219521:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775868.219522:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775868.219525:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775868.219527:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12750 00000400:00000200:3.0:1775775868.219531:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775868.219534:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775868.219536:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775868.219539:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775868.219541:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775868.219542:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775868.219544:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775868.219546:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775868.219547:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775868.219548:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775868.219551:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775868.219553:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775868.219555:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775868.219556:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775868.219559:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775868.219560:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775868.219561:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775868.219565:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775868.219569:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775868.219570:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775868.219572:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775868.219582:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775868.219584:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775868.219586:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775868.219590:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775868.219594:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775868.219599:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775868.219608:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000020:00000040:3.0:1775775868.219614:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775868.219624:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775868.219627:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775868.219634:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775868.219637:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775868.219642:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775868.219649:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000100:00000001:3.0:1775775868.219659:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775868.219671:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710371712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775868.219681:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775868.219684:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775868.219691:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710371712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775868.219702:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775868.219706:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775868.219711:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775868.219713:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775868.219715:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775868.219718:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775868.219723:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12750 00000400:00000200:3.0:1775775868.219737:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775868.219742:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775868.219746:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775868.219749:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775868.219751:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775868.219777:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775868.219780:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775868.219785:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775868.219787:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775868.219791:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775868.219794:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775868.219800:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775868.219805:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775868.219814:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775868.219821:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775868.219827:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775868.219829:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775868.219833:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775868.219839:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775868.219843:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775868.219847:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775868.219850:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775868.219856:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775868.219858:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775868.219860:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775868.219866:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775868.219870:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775868.219873:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775775868.219878:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775868.219881:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775868.219882:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775868.219884:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775868.219885:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775868.219887:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775868.219891:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000001:3.0:1775775868.219897:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775868.219906:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710371840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775868.219924:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775868.219929:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775868.219934:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710371840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775868.219947:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775868.219951:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775868.219960:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775868.219969:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775868.219984:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775868.219996:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12760) 00000800:00000200:1.1:1775775868.220036:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775868.220039:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775775868.220371:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775868.220374:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775775868.220578:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775868.220580:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00010000:00000001:3.0:1775775868.347213:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775868.347219:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775868.347223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775868.347226:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775868.347228:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775868.347230:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775868.347233:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775868.347235:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775868.347236:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775868.347239:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775868.347242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775868.347243:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775868.347244:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775868.347245:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775868.347246:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775868.347247:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775868.347248:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775868.347249:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775868.347315:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775868.347321:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775868.347327:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775869.371268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775869.371275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775869.371280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775869.371282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775869.371285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775869.371286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775869.371352:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775869.371355:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775869.371358:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775870.395268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775870.395272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775870.395276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775870.395277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775870.395280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775870.395281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775870.395342:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775870.395346:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775870.395348:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775871.419289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775871.419297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775871.419302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775871.419303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775871.419306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775871.419307:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775872.123234:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307427078, at 4307427136 00000100:00000010:0.0:1775775872.123250:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e28a0. 00010000:00000001:3.0:1775775872.443245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775872.443251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775872.443257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775872.443258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775872.443262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775872.443264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775872.443287:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775872.443290:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775872.443293:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775873.339242:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775873.339248:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775873.339251:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775873.339255:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775873.339260:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775873.339267:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12755 00000400:00000200:0.0:1775775873.339276:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775873.339282:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775873.339284:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775873.339287:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775873.339289:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775873.339291:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775873.339293:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775873.339295:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775873.339297:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775873.339299:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775873.339301:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775873.339303:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775873.339304:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775873.339306:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775873.339308:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775873.339309:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775873.339310:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775873.339314:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775873.339316:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775873.339318:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775873.339321:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775873.339323:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775873.339325:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775873.339326:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775873.339328:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775873.339335:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063b80. 00000020:00000040:0.0:1775775873.339338:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775873.339341:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775873.339342:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775873.339345:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775873.339347:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775873.339350:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775873.339356:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5c00. 00000100:00000001:0.0:1775775873.339376:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775873.339384:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063b80 x1862022710371968/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775873.339392:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775873.339395:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775873.339398:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ee9007b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063b80 x1862022710371968/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775873.339414:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775873.339416:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775873.339418:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775873.339419:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775873.339421:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775873.339423:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775873.339425:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12755 00000400:00000200:0.0:1775775873.339429:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775873.339433:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775873.339435:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775873.339437:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775873.339438:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775873.339440:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775873.339442:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775873.339445:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775873.339448:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775873.339450:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775873.339452:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775873.339455:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775873.339457:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775873.339463:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775873.339468:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775873.339472:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775873.339475:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775873.339485:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775873.339489:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775873.339494:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775873.339497:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775873.339502:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775873.339505:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775873.339508:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775873.339511:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775873.339515:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775873.339518:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775873.339523:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061f80. 00000020:00000040:0.0:1775775873.339525:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775873.339527:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775873.339529:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775873.339531:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775873.339534:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775873.339536:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775873.339541:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7c00. 00000100:00000001:0.0:1775775873.339546:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775873.339552:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061f80 x1862022710372096/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775873.339557:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775873.339559:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775873.339568:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000029d3794] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061f80 x1862022710372096/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775873.339582:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775873.339587:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775873.339596:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775873.339600:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775873.339604:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775873.339610:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775873.339616:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12755 00000400:00000200:0.0:1775775873.339623:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775873.339627:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775873.339633:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775873.339639:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775873.339642:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775873.339646:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775873.339650:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775873.339653:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775873.339657:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775873.339660:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775873.339663:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775873.339666:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775873.339670:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775873.339678:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775873.339683:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775873.339690:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775873.339695:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775873.339701:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775873.339705:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775873.339711:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775873.339716:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775873.339720:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775873.339726:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775873.339728:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775873.339732:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775873.339735:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775873.339740:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775873.339746:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060700. 00000020:00000040:0.0:1775775873.339750:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000800:00000200:1.1:1775775873.339750:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775873.339754:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 02000000:00000001:0.0:1775775873.339764:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775873.339766:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775873.339771:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775873.339775:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775873.339779:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775873.339785:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4800. 00000100:00000001:0.0:1775775873.339790:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775873.339800:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060700 x1862022710372224/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775873.339820:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775873.339825:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775873.339835:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000225c8695] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060700 x1862022710372224/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775873.339859:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775873.339863:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775873.339868:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775873.339873:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775873.339880:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775873.339887:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12766) 00000800:00000200:1.1:1775775873.340240:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775873.340242:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775775873.340573:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775873.340575:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00010000:00000001:3.0:1775775873.467251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775873.467259:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775873.467261:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775873.467265:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775873.467270:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775873.467272:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775873.467275:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775873.467277:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775873.467282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775873.467283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775873.467287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775873.467288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775873.467354:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775873.467388:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775873.467392:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775874.491272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775874.491280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775874.491285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775874.491286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775874.491290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775874.491290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775874.491393:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775874.491399:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775874.491404:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.0:1775775875.195251:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775775875.195260:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775775875.195265:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1775775875.195524:0:2405:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:3.0:1775775875.515258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775875.515264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775875.515270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775875.515271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775875.515275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775875.515276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775875.515371:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775875.515375:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775875.515379:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775876.539254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775876.539259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775876.539264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775876.539266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775876.539270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775876.539271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775877.243233:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307432198, at 4307432256 00000100:00000010:0.0:1775775877.243251:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2e40. 00010000:00000001:3.0:1775775877.563215:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775877.563222:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775877.563226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775877.563227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775877.563230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775877.563231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775877.563248:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775877.563250:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775877.563252:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775775878.459219:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775878.459232:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775878.459235:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775878.459238:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775878.459244:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775878.459254:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12761 00000400:00000200:0.0:1775775878.459263:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775878.459285:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775878.459288:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775878.459291:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775878.459293:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775878.459295:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775878.459297:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775878.459300:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775878.459301:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775878.459304:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775878.459306:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775878.459307:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775878.459309:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775878.459312:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775878.459314:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775878.459316:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775878.459317:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775878.459320:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775878.459323:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775878.459325:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775878.459328:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775878.459330:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775878.459332:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775878.459333:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775878.459335:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775878.459343:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061880. 00000020:00000040:0.0:1775775878.459346:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775878.459349:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775878.459351:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775878.459353:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775878.459355:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775878.459359:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775878.459366:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7c00. 00000100:00000001:0.0:1775775878.459371:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775878.459378:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061880 x1862022710372352/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775878.459387:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775878.459391:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775878.459397:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005afd1f27] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061880 x1862022710372352/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775878.459418:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775878.459422:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775878.459426:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775878.459427:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775878.459429:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775878.459433:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775878.459437:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12761 00000400:00000200:0.0:1775775878.459443:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775878.459447:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775878.459449:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775878.459455:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775878.459460:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775878.459467:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775878.459470:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775878.459476:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775878.459479:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775878.459482:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775878.459485:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775878.459488:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775878.459492:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775878.459496:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775878.459499:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775878.459503:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775878.459505:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775878.459509:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775878.459513:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775878.459519:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775878.459521:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775878.459528:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775878.459531:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775878.459533:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775878.459538:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775878.459541:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775878.459542:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775878.459546:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061180. 00000020:00000040:0.0:1775775878.459550:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775878.459554:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775878.459557:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775878.459561:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775878.459564:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775878.459567:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775878.459574:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5c00. 00000100:00000001:0.0:1775775878.459579:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775878.459592:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061180 x1862022710372480/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775878.459624:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775878.459629:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775878.459638:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e23859] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061180 x1862022710372480/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775878.459653:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775878.459657:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775878.459665:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775878.459670:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775878.459672:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775878.459675:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775878.459679:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12761 00000400:00000200:0.0:1775775878.459711:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775878.459719:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775878.459722:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775878.459725:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775878.459730:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775878.459736:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775878.459741:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775878.459743:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775878.459746:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000800:00000200:1.1:1775775878.459746:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:0.0:1775775878.459749:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775878.459751:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000800:00000200:1.1:1775775878.459751:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000020:00000001:0.0:1775775878.459756:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775878.459760:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775878.459766:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775878.459769:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775878.459772:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775878.459774:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775878.459775:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775878.459780:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775878.459783:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775878.459786:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775878.459789:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775878.459792:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775878.459794:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775878.459796:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775878.459799:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775878.459804:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775878.459807:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061c00. 00000020:00000040:0.0:1775775878.459809:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775878.459812:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775878.459814:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775878.459816:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775878.459818:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775878.459820:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775878.459826:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7000. 00000100:00000001:0.0:1775775878.459831:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775878.459840:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061c00 x1862022710372608/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775878.459862:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775878.459864:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775878.459870:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076a44879] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061c00 x1862022710372608/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775878.459876:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775878.459879:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775878.459884:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775878.459890:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775878.459898:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775878.459903:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12771) 00000800:00000200:1.1:1775775878.460201:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775878.460203:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775775878.460415:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775878.460417:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00010000:00000001:3.0:1775775878.587237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775878.587247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775878.587261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775878.587265:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775878.587278:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775878.587283:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775878.587287:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775878.587290:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775878.587293:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775878.587296:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775878.587301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775878.587303:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775878.587304:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775878.587306:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775878.587308:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775878.587310:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775878.587311:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775878.587314:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775878.587379:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775878.587383:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775878.587386:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775879.611222:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775879.611226:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775879.611231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775879.611232:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775879.611235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775879.611236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775879.611312:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775879.611315:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775879.611318:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775880.635260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775880.635266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775880.635270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775880.635271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775880.635274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775880.635275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775880.635341:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775880.635345:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775880.635349:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775881.659232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775881.659240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775881.659245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775881.659246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775881.659249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775881.659250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775882.363245:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307437318, at 4307437376 00000100:00000010:0.0:1775775882.363261:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e27e0. 00010000:00000001:3.0:1775775882.683246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775882.683253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775882.683260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775882.683262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775882.683265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775882.683266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775882.683286:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775882.683288:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775882.683290:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775883.579256:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775883.579263:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775883.579266:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775883.579271:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775883.579274:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775883.579280:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12766 00000400:00000200:3.0:1775775883.579288:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775883.579293:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775883.579295:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775883.579298:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775883.579300:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775883.579302:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775883.579304:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775883.579307:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775883.579308:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775883.579310:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775883.579312:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775883.579314:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775883.579315:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775883.579318:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775883.579320:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775883.579322:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775883.579324:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775883.579327:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775883.579330:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775883.579333:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775883.579336:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775883.579339:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775883.579342:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775883.579345:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775883.579347:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775883.579356:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775775883.579360:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775883.579364:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775883.579366:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775883.579369:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775883.579371:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775883.579375:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775883.579384:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:3.0:1775775883.579391:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775883.579400:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710372736/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775883.579411:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775883.579415:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775883.579420:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710372736/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775883.579440:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775883.579444:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775883.579448:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775883.579449:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775883.579451:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775883.579455:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775883.579459:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12766 00000400:00000200:3.0:1775775883.579466:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775883.579473:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775883.579478:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775883.579486:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775883.579492:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775883.579497:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775883.579502:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775883.579507:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775883.579511:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775883.579515:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775883.579520:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775883.579525:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775883.579530:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775883.579537:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775883.579540:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775883.579547:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775883.579551:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775883.579554:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775883.579559:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775883.579563:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775883.579565:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775883.579573:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775883.579580:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775883.579585:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775883.579591:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775883.579598:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775883.579602:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775883.579609:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775775883.579613:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775883.579615:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775883.579616:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775883.579618:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775883.579620:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775883.579623:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775883.579628:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775775883.579634:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775883.579643:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710372864/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775883.579658:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775883.579662:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775883.579670:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710372864/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775883.579679:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775883.579681:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775883.579685:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775883.579687:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775883.579690:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775883.579695:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775883.579699:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12766 00000400:00000200:3.0:1775775883.579707:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775883.579713:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775883.579717:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775883.579719:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775883.579721:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775883.579723:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775883.579727:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775883.579729:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775883.579730:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775883.579731:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775883.579733:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775883.579735:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775883.579739:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775883.579743:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775883.579746:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775883.579749:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775883.579752:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000200:1.1:1775775883.579755:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000040:3.0:1775775883.579756:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775883.579760:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000800:00000200:1.1:1775775883.579763:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000100:00000001:3.0:1775775883.579764:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775883.579766:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775883.579769:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775883.579772:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775883.579776:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775883.579779:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775883.579782:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775883.579785:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775883.579788:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775775883.579790:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775883.579792:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775883.579795:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775883.579796:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775883.579798:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775883.579801:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775883.579804:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000001:3.0:1775775883.579807:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775883.579811:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710372992/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775883.579821:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775883.579823:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775883.579827:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710372992/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775883.579835:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775883.579839:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775883.579846:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775883.579851:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775883.579855:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775883.579859:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12776) 00000800:00000200:1.1:1775775883.580161:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775883.580163:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775775883.580385:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775883.580387:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00010000:00000001:3.0:1775775883.707264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775883.707271:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775883.707273:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775883.707275:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775883.707279:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775883.707281:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775883.707282:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775883.707284:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775883.707287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775883.707288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775883.707290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775883.707291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775883.707387:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775883.707394:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775883.707399:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775884.731247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775884.731253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775884.731259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775884.731260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775884.731263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775884.731264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775884.731332:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775884.731337:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775884.731342:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775885.755278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775885.755284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775885.755290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775885.755291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775885.755294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775885.755296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775885.755347:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775885.755351:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775885.755355:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775886.779218:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775886.779225:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775886.779230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775886.779231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775886.779235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775886.779236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775887.483215:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307442438, at 4307442496 00000100:00000010:0.0:1775775887.483235:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ba0. 00010000:00000001:3.0:1775775887.803237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775887.803244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775887.803249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775887.803250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775887.803254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775887.803255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775887.803280:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775887.803282:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775887.803285:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775888.699244:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775888.699259:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775888.699263:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775888.699268:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775888.699273:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775888.699282:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12771 00000400:00000200:3.0:1775775888.699297:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775888.699305:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775888.699308:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775888.699311:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775888.699314:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775888.699316:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775888.699318:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775888.699321:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775888.699323:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775888.699325:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775888.699328:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775888.699330:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775888.699331:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775888.699334:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775888.699335:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775888.699337:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775888.699338:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775888.699340:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775888.699343:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775888.699344:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775888.699348:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775888.699351:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775888.699353:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775888.699355:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775888.699357:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775888.699367:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73b80. 00000020:00000040:3.0:1775775888.699370:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775888.699374:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775888.699375:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775888.699379:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775888.699382:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775888.699386:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775888.699395:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:3.0:1775775888.699402:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775888.699412:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73b80 x1862022710373120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775888.699420:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775888.699425:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775888.699430:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c52008f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73b80 x1862022710373120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775888.699448:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775888.699451:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775888.699455:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775888.699456:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775888.699458:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775888.699461:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775888.699465:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12771 00000400:00000200:3.0:1775775888.699470:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775888.699474:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775888.699478:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775888.699485:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775888.699489:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775888.699496:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775888.699502:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775888.699507:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775888.699511:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775888.699515:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775888.699519:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775888.699523:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775888.699528:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775888.699531:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775888.699535:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775888.699537:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775888.699542:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775888.699545:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775888.699552:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775888.699554:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775888.699557:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775888.699565:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775888.699568:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775888.699571:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775888.699576:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775888.699580:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775888.699581:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775888.699586:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775775888.699589:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775888.699591:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775888.699593:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775888.699596:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775888.699598:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775888.699600:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775888.699606:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000001:3.0:1775775888.699610:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775888.699615:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710373248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775888.699628:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775888.699631:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775888.699637:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710373248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775888.699660:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775888.699664:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775888.699669:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775888.699671:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775888.699674:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775888.699678:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775888.699683:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12771 00000400:00000200:3.0:1775775888.699696:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775888.699702:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775888.699705:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775888.699709:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775888.699712:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775888.699717:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775888.699721:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775888.699723:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775888.699726:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775888.699729:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775888.699731:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775888.699734:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775888.699736:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775888.699742:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775888.699745:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775888.699750:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775888.699753:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775888.699756:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775888.699758:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775888.699761:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775888.699764:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775888.699768:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775888.699773:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775888.699777:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775888.699778:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775888.699781:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775888.699783:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775888.699790:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775775888.699794:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775888.699800:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775888.699801:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775888.699803:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775888.699806:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775888.699809:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775888.699816:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000100:00000001:3.0:1775775888.699824:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775888.699830:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00100000:3.0:1775775888.699834:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710373376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1775775888.699835:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000100:00000001:3.0:1775775888.699851:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775888.699854:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775888.699858:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710373376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775888.699865:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775888.699867:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775888.699872:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775888.699878:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775888.699885:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775888.699891:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12781) 00000800:00000200:1.1:1775775888.700244:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775888.700245:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775775888.700440:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775888.700442:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00010000:00000001:3.0:1775775888.827240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775888.827246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775888.827251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775888.827255:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775888.827257:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775888.827261:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775888.827264:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775888.827266:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775888.827268:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775888.827270:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775888.827273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775888.827274:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775888.827275:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775888.827276:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775888.827277:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775888.827278:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775888.827280:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775888.827281:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775888.827358:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775888.827362:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775888.827365:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775889.851271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775889.851277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775889.851284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775889.851285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775889.851289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775889.851291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775889.851378:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775889.851384:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775889.851388:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775890.875254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775890.875259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775890.875263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775890.875264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775890.875266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775890.875267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775890.875363:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775890.875366:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775890.875369:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775891.899249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775891.899255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775891.899258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775891.899259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775891.899262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775891.899263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775892.603241:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307447558, at 4307447616 00000100:00000010:0.0:1775775892.603256:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e20c0. 00010000:00000001:3.0:1775775892.923238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775892.923243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775892.923247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775892.923248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775892.923251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775892.923252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775892.923271:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775892.923273:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775892.923275:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775775893.819268:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775893.819275:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775893.819279:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775893.819282:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775893.819287:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775893.819294:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12776 00000400:00000200:2.0:1775775893.819308:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775893.819314:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775893.819317:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775893.819321:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775893.819323:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775893.819325:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775893.819327:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775893.819329:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775893.819331:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775893.819333:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775893.819335:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775893.819336:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775893.819338:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775893.819340:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775893.819341:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775893.819343:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775893.819344:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775893.819346:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775893.819348:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775893.819350:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775893.819352:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775893.819354:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775893.819356:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775893.819358:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775893.819360:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775893.819368:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775775893.819371:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775893.819374:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775893.819375:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775893.819378:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775893.819380:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775893.819383:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775893.819390:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775775893.819396:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775893.819407:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710373504/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775893.819415:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775893.819419:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775893.819422:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710373504/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775893.819438:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775893.819440:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775893.819443:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775893.819444:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775893.819445:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775893.819447:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775893.819449:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12776 00000400:00000200:2.0:1775775893.819452:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775893.819454:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775893.819456:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775893.819458:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775893.819460:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775893.819461:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775893.819463:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775893.819465:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775893.819467:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775893.819469:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775893.819470:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775893.819472:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775893.819473:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775893.819476:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775893.819481:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775893.819483:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775893.819488:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775893.819494:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775893.819500:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775893.819506:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775893.819511:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775893.819518:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775893.819523:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775893.819528:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775893.819535:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775893.819541:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775893.819545:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775893.819550:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775775893.819553:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775893.819560:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775893.819561:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775893.819565:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775893.819567:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775893.819570:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775893.819575:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775775893.819579:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775893.819586:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710373632/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775893.819597:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775893.819600:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775893.819603:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710373632/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775893.819616:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775893.819620:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775893.819625:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775893.819627:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775893.819629:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775893.819631:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775893.819633:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12776 00000400:00000200:2.0:1775775893.819640:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775893.819642:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775893.819644:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775893.819649:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775893.819650:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775893.819652:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775893.819654:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775893.819660:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775893.819661:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775893.819665:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775893.819666:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775893.819670:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775893.819672:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775893.819677:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775893.819678:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775893.819682:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775893.819686:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775893.819691:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775893.819697:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775893.819700:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775893.819703:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775893.819707:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775893.819709:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775893.819714:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775893.819716:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775893.819721:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775893.819722:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775893.819727:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775775893.819730:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775893.819732:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775893.819733:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775893.819735:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775893.819737:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775893.819739:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775893.819744:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775775893.819747:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775893.819754:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710373760/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775893.819763:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775893.819765:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775893.819771:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710373760/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775893.819790:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775893.819794:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775893.819799:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775893.819802:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775893.819809:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775893.819814:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12786) 00000800:00000200:1.1:1775775893.819865:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775893.819869:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775775893.820344:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775893.820347:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775775893.820677:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775893.820680:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00010000:00000001:3.0:1775775893.947256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775893.947262:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775893.947264:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775893.947267:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775893.947272:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775893.947273:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775893.947275:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775893.947276:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775893.947279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775893.947280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775893.947283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775893.947284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775893.947339:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775893.947342:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775893.947346:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775894.971279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775894.971287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775894.971294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775894.971301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775894.971306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775894.971308:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775894.971399:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775894.971404:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775894.971407:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775895.995259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775895.995264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775895.995268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775895.995269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775895.995272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775895.995273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775895.995341:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775895.995345:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775895.995349:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775897.019203:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775897.019209:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775897.019215:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775897.019216:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775897.019219:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775897.019221:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775897.723244:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307452678, at 4307452736 00000100:00000010:0.0:1775775897.723264:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2b40. 00010000:00000001:3.0:1775775898.043277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775898.043283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775898.043288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775898.043296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775898.043299:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775898.043300:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775898.043335:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775898.043339:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775898.043342:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775898.939297:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775898.939305:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775898.939309:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775898.939311:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775898.939315:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775898.939323:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12781 00000400:00000200:3.0:1775775898.939334:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775898.939340:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775898.939343:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775898.939345:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775898.939348:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775898.939350:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775898.939352:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775898.939356:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775898.939357:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775898.939360:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775898.939362:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775898.939364:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775898.939366:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775898.939368:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775898.939369:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775898.939371:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775898.939373:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775898.939375:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775898.939378:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775898.939380:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775898.939383:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775898.939385:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775898.939388:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775898.939389:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775898.939390:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775898.939409:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775775898.939412:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775898.939416:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775898.939417:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775898.939420:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775898.939422:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775898.939427:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775898.939437:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000100:00000001:3.0:1775775898.939444:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775898.939453:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710373888/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775898.939462:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775898.939467:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775898.939470:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710373888/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775898.939489:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775898.939492:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775898.939494:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775898.939495:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775898.939497:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775898.939499:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775898.939500:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12781 00000400:00000200:3.0:1775775898.939503:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775898.939505:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775898.939507:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775898.939509:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775898.939511:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775898.939513:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775898.939514:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775898.939516:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775898.939518:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775898.939519:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775898.939521:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775898.939522:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775898.939523:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775898.939525:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775898.939527:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775898.939529:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775898.939532:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775898.939536:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775898.939540:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775898.939542:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775898.939544:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775898.939550:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775898.939554:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775898.939556:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775898.939561:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775898.939566:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775898.939569:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775898.939573:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775775898.939575:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775898.939580:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775898.939582:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775898.939583:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775898.939586:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775898.939588:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775898.939591:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000100:00000001:3.0:1775775898.939595:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775898.939604:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710374016/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775898.939616:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775898.939619:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775898.939622:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710374016/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775898.939632:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775898.939635:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775898.939637:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775898.939638:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775898.939640:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775898.939642:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775898.939645:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12781 00000400:00000200:3.0:1775775898.939651:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775898.939655:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775898.939658:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775898.939663:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775898.939664:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775898.939666:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775898.939668:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775898.939672:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775898.939673:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775898.939673:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775898.939675:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775898.939676:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775898.939678:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775898.939681:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775898.939684:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775898.939685:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775898.939686:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775898.939688:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775898.939689:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775898.939691:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775898.939692:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775898.939694:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775898.939695:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775898.939699:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775898.939700:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775898.939704:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775898.939706:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775898.939709:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000020:00000040:3.0:1775775898.939712:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775898.939713:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775898.939715:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775898.939717:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775898.939719:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775898.939721:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775898.939726:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000100:00000001:3.0:1775775898.939730:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775898.939737:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710374144/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775898.939746:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775898.939748:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775898.939754:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710374144/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775898.939764:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775898.939767:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775898.939771:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775898.939774:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775898.939778:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775898.939782:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12791) 00000800:00000200:1.1:1775775898.939845:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775898.939848:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775775898.940207:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775898.940209:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775775898.940489:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775898.940491:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00010000:00000001:3.0:1775775899.067227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775899.067233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775899.067238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775899.067242:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775899.067244:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775899.067247:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775899.067250:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775899.067252:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775899.067254:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775899.067256:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775899.067258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775899.067259:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775899.067260:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775899.067261:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775899.067262:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775899.067263:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775899.067264:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775899.067265:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775899.067364:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775899.067367:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775899.067370:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775900.091266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775900.091274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775900.091278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775900.091279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775900.091283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775900.091285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775900.091383:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775900.091388:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775900.091393:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775901.115276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775901.115283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775901.115289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775901.115290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775901.115294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775901.115297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775901.115378:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775901.115383:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775901.115387:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775902.139295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775902.139305:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775902.139313:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775902.139315:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775902.139319:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775902.139320:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775902.843231:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307457798, at 4307457856 00000100:00000010:0.0:1775775902.843249:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2660. 00010000:00000001:3.0:1775775903.163249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775903.163255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775903.163259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775903.163260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775903.163264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775903.163265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775775903.163284:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775775903.163286:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775775903.163289:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775775904.059226:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775904.059238:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775904.059243:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775904.059248:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775904.059255:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775904.059264:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12786 00000400:00000200:3.0:1775775904.059274:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775904.059281:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775904.059285:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775904.059289:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775904.059292:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775904.059295:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775904.059298:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775904.059302:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775904.059304:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775904.059306:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775904.059310:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775904.059313:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775904.059316:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775904.059320:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775904.059322:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775904.059324:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775904.059327:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775904.059330:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775904.059334:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775904.059337:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775904.059341:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775904.059344:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775904.059347:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775904.059350:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775904.059353:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775904.059362:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775775904.059366:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775904.059370:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775904.059372:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775904.059375:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775904.059378:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775904.059382:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775904.059390:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000100:00000001:3.0:1775775904.059397:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775904.059406:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710374272/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775904.059417:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775904.059422:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775904.059427:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710374272/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775904.059453:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775904.059456:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775904.059460:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775904.059462:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775904.059464:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775904.059468:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775904.059471:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12786 00000400:00000200:3.0:1775775904.059477:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775904.059481:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775904.059484:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775904.059490:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775904.059495:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775904.059500:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775904.059506:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775904.059512:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775904.059515:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775904.059519:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775904.059523:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775904.059528:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775904.059533:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775904.059544:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775904.059549:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775904.059562:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775904.059571:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775904.059574:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775904.059577:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775904.059580:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775904.059583:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775904.059590:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775904.059596:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775904.059599:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775904.059604:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775904.059609:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775904.059614:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775904.059619:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775775904.059623:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775904.059627:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775904.059629:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775904.059633:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775904.059636:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775904.059640:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775904.059645:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000100:00000001:3.0:1775775904.059654:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775904.059664:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710374400/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775904.059678:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775904.059681:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775904.059690:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710374400/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775904.059710:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775904.059716:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775904.059722:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775904.059726:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775904.059729:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775904.059735:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775904.059741:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12786 00000400:00000200:3.0:1775775904.059749:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775904.059759:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775904.059765:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775904.059772:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775904.059775:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775904.059779:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775904.059785:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775904.059790:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775904.059794:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775904.059798:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:1.1:1775775904.059798:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000040:3.0:1775775904.059801:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000800:00000200:1.1:1775775904.059802:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000020:00000001:3.0:1775775904.059807:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775904.059812:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775904.059820:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775904.059841:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775904.059847:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775904.059852:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775904.059856:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775904.059860:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775904.059865:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775904.059870:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775904.059875:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775904.059879:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775904.059883:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775904.059887:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775904.059894:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775904.059898:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775904.059905:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775775904.059911:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775904.059917:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775904.059920:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775904.059923:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775904.059926:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775904.059929:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775904.059935:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000100:00000001:3.0:1775775904.059940:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775904.059947:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710374528/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775904.059960:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775904.059963:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775904.059970:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710374528/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775904.059988:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775904.059992:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775904.059997:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775904.060001:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775904.060004:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775904.060009:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12796) 00000800:00000200:1.1:1775775904.060325:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775904.060327:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775775904.060542:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775904.060544:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00010000:00000001:3.0:1775775904.187257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775904.187265:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775904.187267:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775904.187272:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775904.187277:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775904.187280:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775904.187282:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775904.187284:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775904.187288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775904.187290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775904.187294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775904.187295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775904.187375:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775904.187380:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775904.187384:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775905.211248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775905.211253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775905.211256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775905.211257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775905.211260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775905.211261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775905.211313:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775905.211316:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775905.211319:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775906.235229:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775906.235235:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775906.235241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775906.235242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775906.235245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775906.235245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775906.235309:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775906.235312:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775906.235316:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775907.259263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775907.259272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775907.259280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775907.259282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775907.259286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775907.259287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775907.963263:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307462918, at 4307462976 00000100:00000010:0.0:1775775907.963289:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2de0. 00010000:00000001:3.0:1775775908.283214:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775908.283221:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775908.283228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775908.283229:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775908.283233:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775908.283234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775909.179234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775909.179242:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775909.179244:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775909.179245:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775775909.179252:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:0.0:1775775909.179254:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775909.179256:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775775909.179259:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775775909.179260:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775909.179264:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775775909.179265:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775909.179267:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775775909.179271:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775775909.179271:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775909.179281:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12791 00000400:00000200:0.0:1775775909.179291:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775909.179297:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775909.179301:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775909.179304:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775909.179307:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775909.179309:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775909.179311:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775909.179323:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775909.179326:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775909.179329:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775909.179333:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775909.179336:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775909.179338:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775909.179342:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775909.179344:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775909.179346:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775909.179349:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775909.179352:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775909.179356:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775909.179359:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775909.179363:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775909.179367:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775909.179371:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775909.179374:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775909.179376:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775909.179385:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063100. 00000020:00000040:0.0:1775775909.179388:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775909.179393:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775909.179394:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775909.179397:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775909.179399:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775909.179404:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775909.179419:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6c00. 00000100:00000001:0.0:1775775909.179426:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775909.179436:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063100 x1862022710374656/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775909.179451:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775909.179456:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775909.179461:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033f6db7f] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063100 x1862022710374656/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775909.179484:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775909.179488:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775909.179492:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775909.179494:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775909.179496:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775909.179500:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775909.179503:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12791 00000400:00000200:0.0:1775775909.179516:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775909.179521:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775909.179527:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775909.179532:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775909.179535:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775909.179539:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775909.179541:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775909.179545:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775909.179547:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775909.179551:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775909.179554:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775909.179558:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775909.179562:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775909.179568:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775909.179570:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775909.179572:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775909.179575:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775909.179579:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775909.179582:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775909.179589:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775909.179592:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775909.179599:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775909.179602:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775909.179603:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775909.179607:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775909.179610:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775909.179612:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775909.179616:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060e00. 00000020:00000040:0.0:1775775909.179620:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775909.179624:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775909.179626:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775909.179629:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775909.179632:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775909.179635:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775909.179641:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6400. 00000100:00000001:0.0:1775775909.179647:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775909.179654:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060e00 x1862022710374784/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775909.179666:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775909.179668:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775909.179672:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004a3bddaa] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060e00 x1862022710374784/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775909.179682:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775909.179686:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775909.179691:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775909.179693:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775909.179696:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775909.179700:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775909.179706:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12791 00000400:00000200:0.0:1775775909.179743:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775909.179752:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775909.179756:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775909.179760:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775909.179762:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775909.179766:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775909.179770:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775909.179773:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775909.179776:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775909.179781:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775909.179784:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775909.179789:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:1.1:1775775909.179789:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:0.0:1775775909.179795:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:1.1:1775775909.179796:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000100:00000040:0.0:1775775909.179806:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775909.179811:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775909.179816:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775909.179820:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775909.179826:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775909.179831:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775909.179836:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775909.179839:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775909.179842:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775909.179845:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775909.179847:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775909.179851:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775909.179856:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775909.179860:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775909.179866:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061c00. 00000020:00000040:0.0:1775775909.179869:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775909.179874:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775909.179877:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775909.179880:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775909.179883:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775909.179886:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775909.179890:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4400. 00000100:00000001:0.0:1775775909.179896:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775909.179904:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061c00 x1862022710374912/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775909.179911:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775909.179914:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775909.179916:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076a44879] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061c00 x1862022710374912/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775909.179943:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775909.179949:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775909.179959:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775909.179964:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775909.179968:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775909.179976:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12801) 00000800:00000200:1.1:1775775909.180211:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775909.180214:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775775909.180478:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775909.180481:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00010000:00000001:3.0:1775775909.307244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775909.307251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775909.307257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775909.307262:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775909.307264:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775909.307268:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775909.307272:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775909.307275:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775909.307277:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775909.307280:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775909.307284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775909.307286:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775909.307287:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775909.307289:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775909.307291:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775909.307292:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775909.307294:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775909.307296:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775909.307348:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775909.307351:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775909.307354:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775910.203239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775910.203250:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775910.203252:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775910.203274:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775910.203279:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775910.203283:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775910.203286:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775910.203289:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775910.203292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775910.331275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775910.331281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775910.331286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775910.331287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775910.331290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775910.331291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775910.331347:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775910.331351:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775910.331354:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775911.227230:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775911.227235:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775911.227237:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775911.227242:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775911.227245:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775911.227248:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775911.227250:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775911.227252:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775911.227255:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775911.355269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775911.355275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775911.355281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775911.355282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775911.355287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775911.355288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775911.355362:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775911.355366:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775911.355369:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775912.251278:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775912.251284:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775912.251286:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775912.251292:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775912.251296:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775912.251301:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775912.251304:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775912.251307:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775912.251311:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775912.379271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775912.379276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775912.379282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775912.379283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775912.379287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775912.379289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775913.083242:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307468038, at 4307468096 00000100:00000010:0.0:1775775913.083259:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2300. 00000800:00000200:2.0:1775775913.083265:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775775913.083273:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775775913.083278:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1775775913.083427:0:2405:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1775775913.275269:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775913.275276:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775913.275278:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775913.275283:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775913.275287:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775913.275291:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775913.275293:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775913.275296:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775913.275298:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775913.403254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775913.403260:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775913.403262:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775913.403264:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775913.403268:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775913.403270:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775913.403272:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775913.403273:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775913.403276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775913.403277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775913.403280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775913.403280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775914.299264:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775914.299269:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775914.299271:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775914.299274:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775775914.299275:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775914.299279:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775775914.299284:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775914.299284:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775914.299287:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775775914.299288:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775775914.299290:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775775914.299292:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775914.299293:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775914.299299:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775914.299306:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12796 00000400:00000200:0.0:1775775914.299314:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775914.299327:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775914.299330:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775914.299333:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775914.299336:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775914.299338:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775914.299340:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775914.299343:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775914.299345:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775914.299347:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775914.299350:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775914.299353:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775914.299356:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775914.299359:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775914.299361:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775914.299364:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775914.299366:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775914.299369:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775914.299373:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775914.299375:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775914.299379:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775914.299383:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775914.299386:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775914.299388:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775914.299391:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775914.299401:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061180. 00000020:00000040:0.0:1775775914.299404:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775914.299409:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775914.299411:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775914.299413:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775914.299415:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775914.299418:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775914.299424:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6000. 00000100:00000001:0.0:1775775914.299430:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775914.299438:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061180 x1862022710375040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775914.299446:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775914.299450:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775914.299453:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e23859] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061180 x1862022710375040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775914.299470:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775914.299472:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775914.299475:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775914.299476:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775914.299477:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775914.299479:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775914.299481:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12796 00000400:00000200:0.0:1775775914.299484:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775914.299486:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775914.299488:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775914.299490:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775914.299492:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775914.299494:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775914.299496:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775914.299498:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775914.299499:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775914.299500:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775914.299502:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775914.299503:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775914.299505:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775914.299507:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775914.299511:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775914.299514:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775914.299517:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775914.299520:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775914.299524:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775914.299530:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775914.299534:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775914.299538:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775914.299539:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775914.299540:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775914.299545:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775914.299549:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775914.299552:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775914.299557:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061880. 00000020:00000040:0.0:1775775914.299561:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775914.299566:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775914.299569:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775914.299573:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775914.299576:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775914.299579:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775914.299585:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000001:0.0:1775775914.299590:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775914.299597:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061880 x1862022710375168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775914.299609:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775914.299613:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775914.299616:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005afd1f27] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061880 x1862022710375168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775914.299625:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775914.299628:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775914.299630:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775914.299634:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775914.299635:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775914.299638:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775914.299660:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12796 00000400:00000200:0.0:1775775914.299670:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775914.299677:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775914.299681:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775914.299689:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775914.299694:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775914.299700:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775914.299705:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775914.299710:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775914.299714:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775914.299716:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775914.299719:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775914.299724:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775914.299729:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775914.299738:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775914.299743:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775914.299750:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775914.299755:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775914.299760:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775914.299768:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775914.299774:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775914.299777:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775914.299781:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775914.299785:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775914.299787:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775914.299789:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775914.299794:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775914.299797:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775914.299800:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000010:0.0:1775775914.299801:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060700. 00000020:00000040:0.0:1775775914.299805:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775914.299807:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775914.299808:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775914.299809:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775914.299810:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775914.299812:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775914.299816:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4400. 00000100:00000001:0.0:1775775914.299820:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775914.299827:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060700 x1862022710375296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775914.299841:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775914.299844:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775914.299851:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000225c8695] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060700 x1862022710375296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775914.299869:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775914.299875:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775914.299883:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775914.299887:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775914.299890:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775914.299895:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12807) 00000800:00000200:1.1:1775775914.300033:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775914.300130:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00010000:00000001:3.0:1775775914.427247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775914.427321:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775914.427329:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775914.427330:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775914.427334:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775914.427335:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775914.427390:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775914.427393:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775914.427396:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775915.323305:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775915.323311:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775915.323314:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775915.323320:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775915.323324:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775915.323328:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775915.323330:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775915.323332:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775915.323334:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775915.451259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775915.451263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775915.451267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775915.451269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775915.451272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775915.451273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775915.451331:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775915.451334:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775915.451337:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775916.347271:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775916.347277:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775916.347279:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775916.347283:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775916.347286:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775916.347289:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775916.347291:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775916.347293:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775916.347294:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775916.475225:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775916.475230:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775916.475234:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775916.475235:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775916.475237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775916.475238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775916.475301:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775916.475306:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775916.475309:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775917.371351:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775917.371358:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775917.371360:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775917.371367:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775917.371371:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775917.371375:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775917.371377:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775917.371380:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775917.371382:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1775775917.371498:0:2399:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:0.0:1775775917.371546:0:2399:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1775775917.371558:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775917.371562:0:2399:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:0.0:1775775917.371567:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775917.371572:0:2399:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:0.0:1775775917.371578:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:0.0:1775775917.371582:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:0.0:1775775917.371587:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775775917.371606:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775775917.371615:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3220 00000400:00000010:0.0:1775775917.371618:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3220. 00000100:00000001:0.0:1775775917.371625:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775917.371634:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f622e061180 x1862022710375040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775917.371670:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775917.371675:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061180 x1862022710375040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775775939 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775917.371682:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775917.371684:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775775917.371694:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:0.0:1775775917.371697:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230600. 00000400:00000200:0.0:1775775917.371700:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775775917.371704:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775775917.371707:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f32a8 00000400:00000010:0.0:1775775917.371709:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f32a8. 00000100:00000001:0.0:1775775917.371712:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775917.371717:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f622e061880 x1862022710375168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775917.371727:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775917.371731:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061880 x1862022710375168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775939 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775917.371738:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775917.371740:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775775917.371744:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:0.0:1775775917.371746:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230400. 00000400:00000200:0.0:1775775917.371748:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775775917.371750:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775775917.371752:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3550 00000400:00000010:0.0:1775775917.371753:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3550. 00000100:00000001:0.0:1775775917.371756:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775775917.371761:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f622e060700 x1862022710375296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775939 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775917.371770:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775775917.371772:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e060700 x1862022710375296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775775939 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775917.371779:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775917.371781:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:3.0:1775775917.499244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775917.499249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775917.499253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775917.499254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775917.499257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775917.499258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775918.203242:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307473158, at 4307473216 00000100:00000010:0.0:1775775918.203260:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e25a0. 00000100:00000001:3.0:1775775918.395292:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775918.395298:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775918.395300:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775918.395306:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775918.395310:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775918.395313:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775918.395316:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775918.395318:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775918.395321:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775918.523225:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775918.523229:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775918.523233:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775918.523234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775918.523237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775918.523238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775919.419275:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775775919.419281:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1775775919.419285:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775919.419287:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775775919.419288:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775775919.419290:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775919.419294:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775775919.419296:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775775919.419299:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775775919.419302:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775919.419307:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775775919.419309:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12802 00000100:00000001:3.0:1775775919.419311:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775919.419315:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775919.419318:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775775919.419322:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775919.419338:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775919.419342:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775919.419346:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775919.419350:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775919.419352:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775919.419354:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775919.419359:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775919.419360:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775919.419363:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775919.419366:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775919.419369:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775919.419372:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775919.419376:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775919.419378:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775919.419381:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775919.419383:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775919.419386:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775919.419389:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775919.419391:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775919.419394:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775919.419396:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775919.419398:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775919.419400:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775919.419402:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775919.419408:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775775919.419411:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775919.419414:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775919.419415:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775919.419417:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775919.419419:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775919.419422:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775919.419429:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775775919.419436:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775919.419446:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710375424/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775919.419473:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775919.419477:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775919.419483:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710375424/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775919.419510:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775919.419514:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775919.419518:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775919.419519:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775919.419521:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775919.419524:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775919.419528:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12802 00000400:00000200:2.0:1775775919.419531:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775919.419533:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775919.419535:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775919.419537:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775919.419539:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775919.419540:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775919.419542:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775919.419543:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775919.419544:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775919.419545:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775919.419547:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775919.419548:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775919.419549:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775919.419551:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775919.419552:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775919.419585:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775919.419590:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775919.419595:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775919.419602:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775919.419607:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775919.419613:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775919.419621:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775919.419625:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775919.419630:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775919.419642:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775919.419650:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775919.419653:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775919.419659:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775775919.419663:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775919.419669:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775919.419672:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775919.419674:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775919.419692:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775919.419696:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775919.419702:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775775919.419708:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775919.419716:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710375552/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775919.419726:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775919.419731:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775919.419738:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710375552/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775919.419756:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775919.419762:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775919.419767:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775919.419771:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775919.419774:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775919.419781:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775919.419787:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12802 00000400:00000200:2.0:1775775919.419794:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775919.419812:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775919.419816:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775919.419823:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775919.419827:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775919.419830:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775919.419834:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775919.419838:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775919.419841:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775919.419844:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775919.419849:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775919.419854:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775919.419860:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775919.419868:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775919.419874:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775919.419877:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775919.419880:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775919.419882:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775919.419890:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775919.419897:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775919.419901:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775919.419906:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775919.419910:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775919.419912:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775919.419917:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775919.419921:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775919.419926:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775919.419935:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775775919.419940:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000800:00000200:1.1:1775775919.419941:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:2.0:1775775919.419946:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000800:00000200:1.1:1775775919.419947:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 02000000:00000001:2.0:1775775919.419948:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775919.419953:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775919.419956:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775919.419960:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775919.419966:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775775919.419971:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775919.419980:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710375680/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775919.419993:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775919.419997:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775919.420004:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710375680/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775919.420023:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775919.420030:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775919.420036:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775919.420052:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775919.420058:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775919.420064:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12812) 00000800:00000200:1.1:1775775919.420469:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775919.420471:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775775919.420793:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775919.420795:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00010000:00000001:3.0:1775775919.547244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775919.547249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775919.547253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775919.547256:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775919.547257:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775919.547260:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775919.547262:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775919.547264:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775919.547265:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775919.547267:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775919.547269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775919.547270:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775919.547271:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775919.547272:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775919.547273:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775919.547274:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775919.547275:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775919.547276:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775919.547331:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775919.547334:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775919.547337:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775920.443302:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775920.443309:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775920.443312:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775920.443317:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775920.443321:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775920.443324:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775920.443328:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775920.443330:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775920.443333:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775920.571215:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775920.571220:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775920.571224:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775920.571225:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775920.571227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775920.571228:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775920.571265:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775920.571269:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775920.571273:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775921.467245:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775921.467251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775921.467254:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775921.467260:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775921.467264:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775921.467268:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775921.467271:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775921.467274:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775921.467277:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775921.595241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775921.595256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775921.595265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775921.595267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775921.595271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775921.595273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775921.595333:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775921.595336:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775921.595339:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775922.491246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775922.491254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775922.491256:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775922.491262:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775922.491266:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775922.491271:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775922.491274:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775922.491277:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775922.491280:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775922.619259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775922.619264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775922.619268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775922.619269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775922.619272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775922.619273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775923.323247:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307478278, at 4307478336 00000100:00000010:0.0:1775775923.323255:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2120. 00000100:00000001:3.0:1775775923.515244:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775923.515251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775923.515253:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775923.515258:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775923.515262:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775923.515266:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775923.515269:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775923.515271:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775923.515274:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775923.643217:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775923.643224:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775923.643226:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775923.643229:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775923.643233:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775923.643241:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775923.643243:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775923.643244:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775923.643247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775923.643248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775923.643251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775923.643252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775924.539226:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775775924.539226:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775924.539233:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1775775924.539235:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775924.539236:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775924.539238:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1775775924.539242:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:2.0:1775775924.539243:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775924.539249:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775924.539250:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775775924.539253:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775924.539258:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1775775924.539259:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12807 00000100:00000001:3.0:1775775924.539262:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775924.539266:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775775924.539271:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775924.539279:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775924.539282:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775924.539286:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775924.539290:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775924.539293:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775924.539296:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775924.539301:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775924.539303:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775924.539306:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775924.539309:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775924.539312:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775924.539315:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775924.539318:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775924.539320:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775924.539322:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775924.539324:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775924.539327:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775924.539330:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775924.539334:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775924.539337:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775924.539341:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775924.539344:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775924.539347:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775924.539349:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775924.539357:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775775924.539360:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775924.539364:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775924.539366:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775924.539370:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775924.539372:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775924.539376:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775924.539384:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775775924.539390:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775924.539400:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710375808/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775924.539412:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775924.539416:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775924.539421:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710375808/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775924.539445:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775924.539448:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775924.539453:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775924.539454:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775924.539456:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775924.539479:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775924.539483:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12807 00000400:00000200:2.0:1775775924.539488:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775924.539492:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775924.539495:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775924.539499:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775924.539504:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775924.539509:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775924.539513:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775924.539518:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775924.539522:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775924.539526:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775924.539530:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775924.539536:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775924.539541:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775924.539550:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775924.539555:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775924.539561:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775924.539566:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775924.539571:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775924.539578:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775924.539581:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775924.539585:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775924.539593:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775924.539598:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775924.539601:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775924.539606:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775924.539613:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775924.539617:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775924.539623:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775775924.539629:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775924.539636:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775924.539640:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775924.539645:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775924.539647:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775924.539650:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775924.539657:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775775924.539662:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775924.539673:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710375936/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775924.539690:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775924.539695:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775924.539706:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710375936/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775924.539729:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775924.539734:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775924.539743:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775924.539746:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775924.539749:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775924.539755:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775924.539762:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12807 00000800:00000200:1.1:1775775924.539767:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000400:00000200:2.0:1775775924.539772:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000200:1.1:1775775924.539774:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000400:00000200:2.0:1775775924.539779:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775924.539785:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775924.539792:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775924.539796:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775924.539799:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775924.539806:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775924.539812:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775924.539815:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775924.539818:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775924.539821:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775924.539826:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775924.539829:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775924.539837:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775924.539843:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775924.539849:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775924.539853:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775924.539861:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775924.539869:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775924.539873:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775924.539878:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775924.539883:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775924.539887:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775924.539891:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775924.539894:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775924.539910:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775924.539915:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775924.539924:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775775924.539929:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775924.539936:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775924.539939:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775924.539943:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775924.539947:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775924.539951:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775924.539958:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775775924.539965:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775924.539977:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710376064/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775924.540026:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775924.540031:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775924.540043:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710376064/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775924.540068:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775924.540074:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775924.540083:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775924.540088:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775924.540094:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775924.540102:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12817) 00000800:00000200:1.1:1775775924.540327:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775924.540330:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775775924.540667:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775924.540669:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00010000:00000001:3.0:1775775924.667243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775924.667248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775924.667252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775924.667253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775924.667257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775924.667258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775924.667336:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775924.667339:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775924.667343:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775925.563248:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775925.563254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775925.563255:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775925.563260:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775925.563263:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775925.563267:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775925.563270:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775925.563273:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775925.563276:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775925.691228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775925.691235:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775925.691241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775925.691243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775925.691247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775925.691249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775925.691308:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775925.691312:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775925.691315:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775926.587240:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775926.587246:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775926.587248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775926.587252:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775926.587261:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775926.587264:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775926.587266:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775926.587270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775926.587273:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775926.715273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775926.715279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775926.715285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775926.715286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775926.715291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775926.715292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775926.715361:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775926.715365:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775926.715369:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775927.611242:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775927.611249:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775927.611252:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775927.611257:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775927.611261:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775927.611265:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775927.611268:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775927.611270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775927.611273:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775927.739255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775927.739261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775927.739267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775927.739268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775927.739273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775927.739274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775928.443228:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307483398, at 4307483456 00000100:00000010:0.0:1775775928.443242:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2f60. 00000100:00000001:3.0:1775775928.635254:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775928.635260:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775928.635261:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775928.635266:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775928.635269:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775928.635272:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775928.635274:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775928.635276:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775928.635278:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775928.763295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775928.763301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775928.763308:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775928.763309:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775928.763314:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775928.763315:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775929.659253:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775775929.659259:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775929.659265:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1775775929.659266:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775929.659268:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775929.659270:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1775775929.659275:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:2.0:1775775929.659276:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775929.659282:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775775929.659283:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775775929.659287:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775929.659291:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1775775929.659293:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12812 00000100:00000001:3.0:1775775929.659294:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775929.659300:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775775929.659308:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775929.659315:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775929.659319:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775929.659322:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775929.659326:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775929.659328:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775929.659330:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775929.659334:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775929.659335:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775929.659337:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775929.659340:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775929.659341:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775929.659342:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775929.659344:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775929.659345:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775929.659346:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775929.659347:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775929.659349:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775929.659351:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775929.659353:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775929.659356:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775929.659358:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775929.659360:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775929.659362:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775929.659363:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775929.659374:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775775929.659388:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775929.659393:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775929.659395:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775929.659398:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775929.659401:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775929.659406:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775929.659416:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775929.659424:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775929.659435:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710376192/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775929.659447:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775929.659452:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775929.659456:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710376192/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775929.659479:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775929.659481:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775929.659484:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775929.659485:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775929.659486:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775929.659488:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775929.659490:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12812 00000400:00000200:2.0:1775775929.659493:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775929.659495:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775929.659497:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775929.659499:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775929.659500:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775929.659502:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775929.659503:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775929.659505:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775929.659506:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775929.659507:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775929.659509:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775929.659510:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775929.659511:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775929.659513:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775929.659515:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775929.659516:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775929.659518:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775929.659522:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775929.659524:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775929.659528:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775929.659530:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775929.659536:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775929.659540:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775929.659543:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775929.659548:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775929.659553:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775929.659555:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775929.659561:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775775929.659565:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775929.659568:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775929.659570:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775929.659573:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775929.659575:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775929.659577:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775929.659580:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775929.659584:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775929.659592:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710376320/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775929.659598:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775929.659599:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775929.659604:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710376320/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775929.659612:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775929.659617:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775929.659621:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775929.659622:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775929.659623:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775929.659625:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775929.659629:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12812 00000400:00000200:2.0:1775775929.659631:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775929.659633:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775929.659634:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775929.659637:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775929.659640:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775929.659641:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775929.659643:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775929.659644:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775929.659645:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775929.659646:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775929.659647:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775929.659649:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775929.659654:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775929.659656:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775929.659658:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775929.659661:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775929.659664:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775929.659665:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775929.659667:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775929.659670:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775929.659671:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775929.659673:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775929.659677:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775929.659680:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775929.659682:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775929.659687:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775929.659689:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775929.659693:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775775929.659696:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775929.659699:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775929.659700:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775929.659702:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775929.659704:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775929.659706:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775929.659709:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775775929.659712:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775929.659718:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710376448/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775929.659721:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775929.659723:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775929.659727:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710376448/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775929.659732:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775929.659734:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775929.659738:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775929.659742:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775929.659746:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775929.659754:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12822) 00000800:00000200:1.1:1775775929.659805:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775929.659808:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775775929.660172:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775929.660174:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775775929.660399:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775929.660401:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00010000:00000001:3.0:1775775929.787299:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775929.787307:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775929.787316:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775929.787320:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775929.787323:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775929.787328:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775929.787331:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775929.787334:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775929.787336:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775929.787339:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775929.787343:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775929.787345:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775929.787346:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775929.787348:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775929.787349:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775929.787351:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775929.787352:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775929.787354:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775929.787426:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775929.787431:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775929.787434:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775930.683259:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775930.683269:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775930.683271:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775930.683278:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775930.683281:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775930.683285:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775930.683289:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775930.683291:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775930.683293:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775930.811281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775930.811288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775930.811292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775930.811293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775930.811296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775930.811297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775930.811380:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775930.811385:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775930.811389:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775931.707246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775931.707254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775931.707256:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775931.707262:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775931.707265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775931.707270:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775931.707273:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775931.707275:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775931.707278:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775931.835283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775931.835289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775931.835295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775931.835296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775931.835301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775931.835302:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775931.835390:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775931.835395:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775931.835398:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775932.731266:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775932.731272:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775932.731275:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775932.731286:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775932.731291:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775932.731294:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775932.731297:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775932.731300:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775932.731303:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775932.859366:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775932.859372:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775932.859379:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775932.859380:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775932.859384:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775932.859386:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775933.563260:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307488518, at 4307488576 00000100:00000010:0.0:1775775933.563289:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2a80. 00000100:00000001:3.0:1775775933.755267:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775933.755274:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775933.755276:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775933.755282:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775933.755287:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775933.755291:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775933.755294:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775933.755297:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775933.755300:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775933.883275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775933.883283:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775933.883285:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775933.883288:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775933.883293:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775933.883295:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775933.883296:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775933.883298:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775933.883300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775933.883301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775933.883315:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775933.883316:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775934.779263:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775934.779270:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775934.779272:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775934.779277:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:2.0:1775775934.779278:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775775934.779280:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775934.779284:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775775934.779300:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775934.779303:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775775934.779304:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775775934.779306:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775934.779310:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775934.779311:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775934.779316:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775934.779324:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12817 00000400:00000200:2.0:1775775934.779335:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775934.779342:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775934.779345:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775934.779348:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775934.779351:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775934.779353:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775934.779355:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775934.779358:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775934.779359:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775934.779362:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775934.779364:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775934.779365:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775934.779367:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775934.779369:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775934.779370:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775934.779371:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775934.779372:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775934.779374:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775934.779377:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775934.779379:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775934.779381:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775934.779384:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775934.779386:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775934.779387:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775934.779389:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775934.779397:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775775934.779400:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775934.779403:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775934.779405:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775934.779407:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775934.779410:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775934.779414:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775934.779420:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775775934.779426:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775934.779433:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710376576/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775934.779441:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775934.779445:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775934.779448:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710376576/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775934.779472:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775934.779475:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775934.779479:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775934.779481:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775934.779483:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775934.779486:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775934.779489:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12817 00000400:00000200:2.0:1775775934.779494:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775934.779498:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775934.779500:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775934.779504:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775934.779507:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775934.779509:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775934.779511:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775934.779512:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775934.779514:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775934.779515:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775934.779517:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775934.779521:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775934.779525:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775934.779529:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775934.779531:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775934.779535:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775934.779539:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775934.779543:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775934.779549:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775934.779554:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775934.779556:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775934.779562:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775934.779566:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775934.779568:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775934.779574:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775934.779579:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775934.779582:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775934.779586:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775775934.779589:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775934.779592:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775934.779594:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775934.779596:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775934.779597:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775934.779600:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775934.779606:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775775934.779612:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775934.779619:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710376704/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775934.779628:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775934.779630:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775934.779636:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710376704/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775934.779648:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775934.779654:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775934.779661:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775934.779664:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775934.779667:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775934.779671:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775934.779676:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12817 00000400:00000200:2.0:1775775934.779681:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775934.779686:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775934.779687:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775934.779689:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775934.779690:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775934.779692:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775934.779693:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775934.779695:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775934.779696:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775934.779699:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775934.779702:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775934.779704:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775934.779706:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775934.779709:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775934.779711:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775934.779715:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775934.779716:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775934.779720:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775934.779726:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775934.779729:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775934.779731:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775934.779735:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775934.779738:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775934.779742:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775934.779745:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775934.779749:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775934.779753:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775934.779759:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775775934.779763:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775934.779765:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775934.779768:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775934.779771:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775934.779774:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775934.779776:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775934.779780:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775775934.779784:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775934.779790:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710376832/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775934.779797:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775934.779798:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775934.779804:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710376832/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775934.779814:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775934.779818:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775934.779821:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775934.779826:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775934.779831:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775775934.779831:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775775934.779837:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12827) 00000800:00000200:1.1:1775775934.779837:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775775934.780243:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775934.780246:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775775934.780557:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775934.780560:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00010000:00000001:3.0:1775775934.907242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775934.907246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775934.907251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775934.907252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775934.907255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775934.907256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775934.907337:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775934.907342:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775934.907345:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775935.803246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775935.803253:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775935.803255:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775935.803261:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775935.803266:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775935.803297:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775935.803300:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775935.803303:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775935.803306:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775935.931228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775935.931233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775935.931241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775935.931242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775935.931245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775935.931246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775935.931318:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775935.931322:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775935.931326:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775936.827209:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775936.827215:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775936.827217:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775936.827221:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775936.827224:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775936.827227:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775936.827229:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775936.827231:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775936.827233:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775936.955238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775936.955243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775936.955246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775936.955247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775936.955262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775936.955263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775936.955320:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775936.955323:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775936.955325:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775775937.851231:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775937.851251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775937.851253:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775937.851258:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775937.851261:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775937.851264:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775937.851266:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775937.851269:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775937.851271:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775937.979253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775937.979259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775937.979265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775937.979267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775937.979271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775937.979272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775938.683256:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307493638, at 4307493696 00000100:00000010:0.0:1775775938.683267:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2720. 00000100:00000001:3.0:1775775938.875272:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775775938.875277:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775775938.875279:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775938.875283:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775775938.875287:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775775938.875290:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775938.875292:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775775938.875294:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775775938.875296:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775939.003279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775939.003285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775939.003291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775939.003292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775939.003296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775939.003298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775939.899302:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775939.899311:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775939.899313:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775939.899317:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775939.899321:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775939.899327:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12822 00000400:00000200:3.0:1775775939.899338:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775939.899346:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775939.899351:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775939.899356:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775939.899360:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775939.899363:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775939.899365:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775939.899368:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775939.899370:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775939.899372:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775939.899374:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775939.899376:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775939.899378:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775939.899380:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775939.899381:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775939.899382:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775939.899383:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775939.899385:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775939.899387:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775939.899389:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775939.899391:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775939.899393:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775939.899395:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775939.899397:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775939.899398:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775939.899405:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775775939.899407:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775939.899410:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775939.899411:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775939.899413:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775939.899415:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775939.899418:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775939.899424:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000001:3.0:1775775939.899429:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775939.899436:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710376960/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775939.899444:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775939.899447:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775939.899451:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710376960/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775939.899471:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775939.899473:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775939.899475:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775939.899476:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775939.899477:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775939.899480:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775939.899482:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12822 00000400:00000200:3.0:1775775939.899485:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775939.899488:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775939.899490:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775939.899494:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775939.899496:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775939.899499:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775939.899502:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775939.899504:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775939.899506:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775939.899508:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775939.899510:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775939.899513:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775939.899517:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775939.899523:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775939.899527:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775939.899530:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775939.899533:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775939.899537:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775939.899541:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775939.899546:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775939.899551:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775939.899556:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775939.899562:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775939.899564:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775939.899570:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775939.899574:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775939.899579:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775939.899582:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000020:00000040:3.0:1775775939.899586:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775939.899589:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775939.899591:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775939.899593:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775939.899595:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775939.899597:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775939.899600:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:3.0:1775775939.899603:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775939.899610:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710377088/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775939.899630:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775939.899634:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775939.899637:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710377088/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775939.899646:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775939.899648:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775939.899653:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775939.899654:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775939.899655:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775939.899657:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775939.899661:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12822 00000400:00000200:3.0:1775775939.899666:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775939.899669:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775939.899673:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775939.899678:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775939.899680:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775939.899684:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775939.899686:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775939.899687:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775939.899688:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775939.899689:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775939.899691:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775939.899692:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775939.899693:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775939.899695:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775939.899696:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775939.899700:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775939.899705:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775939.899708:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775939.899713:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775939.899715:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775939.899718:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775939.899720:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775939.899723:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775939.899726:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775939.899727:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775939.899729:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775939.899733:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775939.899737:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775775939.899741:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775939.899742:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775939.899743:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775939.899744:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775939.899745:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775939.899748:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775939.899753:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000100:00000001:3.0:1775775939.899756:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775939.899761:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710377216/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775939.899770:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775939.899773:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775939.899778:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710377216/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775939.899787:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775939.899790:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775939.899797:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775939.899803:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775939.899811:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775939.899818:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12832) 00000800:00000200:1.1:1775775939.899868:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775939.899872:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775775939.900355:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775939.900358:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775775939.900690:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775939.900693:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00010000:00000001:3.0:1775775940.027281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775940.027288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775940.027293:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775940.027297:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775940.027299:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775940.027303:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775940.027306:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775940.027309:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775940.027312:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775940.027315:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775940.027320:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775940.027322:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775940.027324:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775940.027326:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775940.027328:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775940.027330:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775940.027332:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775940.027334:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775940.027402:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775940.027406:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775940.027409:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775941.051238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775941.051243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775941.051247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775941.051248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775941.051251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775941.051252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775941.051307:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775941.051310:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775941.051313:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775942.075265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775942.075270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775942.075274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775942.075275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775942.075278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775942.075279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775942.075360:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775942.075364:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775942.075368:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775943.099241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775943.099247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775943.099254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775943.099256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775943.099259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775943.099261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775943.803281:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307498758, at 4307498816 00000100:00000010:0.0:1775775943.803301:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2600. 00010000:00000001:3.0:1775775944.123271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775944.123280:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775944.123282:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775944.123286:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775944.123291:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775944.123294:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775944.123296:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775944.123298:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775944.123302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775944.123304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775944.123308:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775944.123309:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775945.019261:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775945.019276:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775945.019280:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775945.019285:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775945.019291:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775945.019301:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12827 00000400:00000200:2.0:1775775945.019311:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775945.019317:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775945.019321:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775945.019325:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775945.019328:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775945.019331:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775945.019334:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775945.019338:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775945.019340:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775945.019343:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775945.019345:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775945.019348:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775945.019351:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775945.019354:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775945.019357:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775945.019358:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775945.019361:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775945.019364:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775945.019368:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775945.019370:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775945.019374:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775945.019377:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775945.019379:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775945.019382:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775945.019384:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775945.019394:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775775945.019397:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775945.019402:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775945.019403:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775945.019407:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775945.019409:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775945.019413:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775945.019421:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775945.019428:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775945.019437:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710377344/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775945.019448:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775945.019452:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775945.019457:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710377344/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775945.019479:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775945.019483:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775945.019486:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775945.019487:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775945.019489:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775945.019492:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775945.019496:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12827 00000400:00000200:2.0:1775775945.019506:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775945.019511:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775945.019514:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775945.019521:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775945.019526:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775945.019532:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775945.019538:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775945.019543:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775945.019547:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775945.019550:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775945.019554:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775945.019558:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775945.019563:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775945.019571:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775945.019575:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775945.019580:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775945.019584:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775945.019590:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775945.019596:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775945.019601:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775945.019605:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775945.019612:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775945.019616:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775945.019621:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775945.019626:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775945.019633:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775945.019638:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775945.019646:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775775945.019651:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775945.019656:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775945.019660:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775945.019663:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775945.019667:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775945.019670:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775945.019676:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775945.019681:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775945.019691:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710377472/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775945.019704:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775945.019707:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775945.019715:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710377472/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775945.019735:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775945.019740:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775945.019745:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775945.019747:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775945.019750:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775945.019756:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775945.019763:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12827 00000400:00000200:2.0:1775775945.019771:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775945.019777:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775945.019780:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775945.019787:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775945.019792:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1775775945.019796:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775775945.019798:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:1.1:1775775945.019800:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000100:00000001:2.0:1775775945.019802:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775945.019806:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775945.019810:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775945.019812:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775945.019815:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775945.019820:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775945.019824:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775945.019829:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775945.019833:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775945.019839:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775945.019843:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775945.019846:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775945.019852:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775945.019856:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775945.019860:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775945.019865:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775945.019870:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775945.019875:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775945.019879:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775945.019884:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775945.019888:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775945.019894:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775775945.019900:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775945.019904:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775945.019906:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775945.019909:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775945.019911:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775945.019915:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775945.019921:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775775945.019927:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775945.019936:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710377600/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775945.019951:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775945.019956:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775945.019965:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710377600/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775945.019984:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775945.019990:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775945.019995:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775945.020000:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775945.020007:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775945.020017:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12837) 00000800:00000200:1.1:1775775945.020234:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775945.020236:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775775945.020468:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775945.020470:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00010000:00000001:3.0:1775775945.147231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775945.147236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775945.147241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775945.147242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775945.147244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775945.147245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775945.147309:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775945.147313:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775945.147317:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775946.171256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775946.171262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775946.171269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775946.171271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775946.171275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775946.171276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775946.171357:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775946.171361:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775946.171365:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775947.195235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775947.195241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775947.195247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775947.195248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775947.195252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775947.195253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775947.195314:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775947.195317:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775947.195319:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775948.219239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775948.219245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775948.219252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775948.219253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775948.219258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775948.219259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775948.923240:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307503878, at 4307503936 00000100:00000010:0.0:1775775948.923261:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2420. 00010000:00000001:3.0:1775775949.243247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775949.243252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775949.243258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775949.243259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775949.243263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775949.243265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775950.139242:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775950.139254:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775950.139257:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775950.139261:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775950.139266:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775950.139276:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12832 00000400:00000200:2.0:1775775950.139289:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775950.139296:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775950.139300:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775950.139304:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775950.139307:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775950.139310:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775950.139313:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775950.139316:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775950.139318:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775950.139321:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775950.139325:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775950.139328:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775950.139330:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775950.139333:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775950.139335:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775950.139337:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775950.139339:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775950.139342:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775950.139346:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775950.139348:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775950.139351:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775950.139355:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775950.139357:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775950.139360:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775950.139363:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775950.139372:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775775950.139376:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775950.139380:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775950.139382:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775950.139385:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775950.139387:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775950.139391:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775950.139399:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775950.139406:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775950.139416:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710377728/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775950.139437:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775950.139441:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775950.139447:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710377728/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775950.139468:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775950.139472:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775950.139476:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775950.139478:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775950.139480:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775950.139483:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775950.139486:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12832 00000400:00000200:2.0:1775775950.139492:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775950.139497:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775950.139501:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775950.139507:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775950.139511:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775950.139518:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775950.139524:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775950.139530:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775950.139533:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775950.139538:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775950.139542:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775950.139547:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775950.139553:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775950.139561:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775950.139564:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775950.139568:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775950.139574:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775950.139579:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775950.139584:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775950.139588:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775950.139593:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775950.139601:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775950.139606:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775950.139611:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775950.139616:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775950.139624:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775950.139628:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775950.139637:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775775950.139642:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775950.139649:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775950.139652:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775950.139656:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775950.139657:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775950.139661:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775950.139669:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775775950.139674:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775950.139684:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710377856/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775950.139699:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775950.139704:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775950.139712:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710377856/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775950.139734:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775950.139739:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775950.139743:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775950.139747:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775950.139750:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775950.139758:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775950.139765:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12832 00000400:00000200:2.0:1775775950.139772:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775950.139780:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775950.139785:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775950.139791:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000800:00000200:1.1:1775775950.139793:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:2.0:1775775950.139795:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1775775950.139796:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000100:00000040:2.0:1775775950.139800:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775950.139807:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775950.139812:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775950.139814:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775950.139818:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775950.139822:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775950.139826:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775950.139829:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775950.139835:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775950.139838:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775950.139841:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775950.139844:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775950.139848:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775950.139855:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775950.139861:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775950.139866:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775950.139870:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775950.139874:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775950.139878:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775950.139881:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775950.139888:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775950.139894:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775950.139900:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775775950.139904:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775950.139910:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775950.139912:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775950.139916:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775950.139919:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775950.139922:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775950.139928:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775775950.139934:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775950.139941:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710377984/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775950.139976:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775950.139980:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775950.139988:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710377984/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775950.140004:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775950.140008:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775950.140018:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775950.140051:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775950.140057:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775950.140065:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12842) 00000800:00000200:1.1:1775775950.140325:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775950.140327:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775775950.140604:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775950.140606:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00010000:00000001:3.0:1775775950.267261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775950.267267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775950.267273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775950.267277:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775950.267280:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775950.267283:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775950.267286:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775950.267289:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775950.267291:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775950.267294:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775950.267298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775950.267300:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775950.267301:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775950.267303:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775950.267304:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775950.267306:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775950.267308:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775950.267310:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775950.267369:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775950.267372:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775950.267375:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775951.291274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775951.291280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775951.291286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775951.291288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775951.291292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775951.291293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775951.291346:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775951.291349:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775951.291352:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.0:1775775951.995281:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775775951.995289:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775775951.995294:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1775775951.995464:0:2405:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1775775951.995469:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775775951.995479:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775775951.995483:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775775951.995485:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1775775952.315266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775952.315273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775952.315278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775952.315280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775952.315284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775952.315285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775952.315347:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775952.315350:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775952.315353:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775953.339263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775953.339271:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775953.339274:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775953.339277:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775953.339281:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775953.339284:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775953.339286:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775953.339289:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775953.339294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775953.339297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775953.339301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775953.339303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775954.043230:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307508998, at 4307509056 00000100:00000010:0.0:1775775954.043248:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2360. 00010000:00000001:3.0:1775775954.363252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775954.363257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775954.363261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775954.363262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775954.363265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775954.363266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775955.259232:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775955.259241:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775955.259245:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775955.259249:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775955.259254:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775955.259261:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12837 00000400:00000200:2.0:1775775955.259273:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775955.259287:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775955.259290:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775955.259296:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775955.259298:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775955.259301:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775955.259303:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775955.259306:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775955.259308:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775955.259310:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775955.259312:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775955.259314:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775955.259316:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775955.259318:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775955.259319:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775955.259321:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775955.259322:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775955.259324:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775955.259326:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775955.259328:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775955.259330:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775955.259333:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775955.259334:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775955.259336:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775955.259337:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775955.259345:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775775955.259347:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775955.259350:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775955.259351:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775955.259353:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775955.259355:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775955.259358:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775955.259365:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775775955.259371:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775955.259378:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710378112/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775955.259385:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775955.259389:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775955.259392:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710378112/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775955.259408:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775955.259411:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775955.259413:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775955.259414:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775955.259415:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775955.259418:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775955.259420:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12837 00000400:00000200:2.0:1775775955.259423:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775955.259426:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775955.259428:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775955.259430:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775955.259432:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775955.259435:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775955.259439:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775955.259442:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775955.259443:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775955.259446:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775955.259448:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775955.259452:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775955.259458:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775955.259464:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775955.259469:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775955.259471:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775955.259474:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775955.259479:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775955.259483:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775955.259487:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775955.259490:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775955.259496:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775955.259499:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775955.259502:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775955.259506:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775955.259508:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775955.259511:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775955.259515:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775775955.259517:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775955.259518:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775955.259519:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775955.259520:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775955.259521:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775955.259522:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775955.259528:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775775955.259533:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775955.259538:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710378240/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775955.259548:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775955.259550:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775955.259552:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710378240/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775955.259563:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775955.259567:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775955.259569:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775955.259573:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775955.259574:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775955.259579:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775955.259583:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12837 00000400:00000200:2.0:1775775955.259591:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775955.259595:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775955.259598:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775955.259603:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775955.259607:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775955.259610:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775955.259613:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775955.259618:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775955.259619:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775775955.259620:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775955.259621:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775955.259625:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775955.259628:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775955.259632:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775955.259633:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775955.259638:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775955.259640:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775955.259642:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775955.259647:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775955.259651:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775955.259653:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775955.259657:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775955.259660:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775955.259664:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775955.259667:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775955.259671:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775955.259674:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775955.259678:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775775955.259690:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775955.259695:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775955.259697:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775955.259701:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775955.259703:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775955.259705:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775955.259708:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775775955.259711:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775955.259718:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710378368/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775955.259731:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775955.259734:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775955.259736:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775775955.259740:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710378368/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1775775955.259741:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 23 secs later) 00000100:00000001:2.0:1775775955.259754:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775955.259757:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775955.259760:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775955.259762:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775955.259766:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775955.259772:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12847) 00000800:00000200:1.1:1775775955.260161:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775955.260163:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 23 secs later) 00000800:00000200:1.1:1775775955.260382:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775955.260383:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 23 secs later) 00010000:00000001:3.0:1775775955.387231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775955.387237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775955.387240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775955.387242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775955.387244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775955.387245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775955.387307:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775955.387311:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775955.387314:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775956.411254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775956.411259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775956.411263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775956.411264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775956.411268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775956.411270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775956.411344:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775956.411349:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775956.411353:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775957.435237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775957.435244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775957.435250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775957.435252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775957.435256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775957.435257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775957.435331:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775957.435335:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775957.435337:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775958.459269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775958.459275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775958.459280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775958.459281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775958.459284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775958.459285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775959.163248:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307514118, at 4307514176 00000100:00000010:0.0:1775775959.163267:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e22a0. 00010000:00000001:3.0:1775775959.483222:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775959.483228:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775959.483238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775959.483240:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775959.483242:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775959.483245:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775959.483247:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775959.483249:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775959.483250:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775959.483252:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775959.483255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775959.483257:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775959.483258:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775959.483260:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775959.483261:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775959.483263:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775959.483264:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775959.483267:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775960.379248:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775960.379259:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775775960.379263:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775960.379268:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775960.379276:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775960.379285:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12842 00000400:00000200:2.0:1775775960.379296:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775960.379304:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775960.379308:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775960.379312:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775960.379325:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775960.379328:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775960.379331:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775960.379335:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775960.379336:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775775960.379339:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775960.379342:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775775960.379345:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775775960.379347:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775775960.379350:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775960.379352:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775960.379354:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775960.379356:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775960.379359:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775775960.379362:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775960.379364:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775960.379367:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775960.379371:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775960.379374:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775960.379376:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775775960.379379:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775960.379388:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775775960.379391:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775775960.379395:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775960.379397:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775960.379400:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775960.379402:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775960.379407:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775960.379414:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775775960.379425:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775960.379435:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710378496/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775960.379444:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775960.379448:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775960.379453:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710378496/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775960.379473:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775960.379476:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775960.379479:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775775960.379481:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775960.379482:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775960.379485:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775960.379488:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12842 00000400:00000200:2.0:1775775960.379494:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775960.379498:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775960.379501:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775960.379506:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775960.379512:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775960.379516:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775960.379520:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775775960.379524:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775960.379527:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775775960.379530:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775960.379534:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775775960.379539:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775775960.379544:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775775960.379553:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775960.379557:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775960.379562:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775960.379566:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775960.379572:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775775960.379580:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775960.379586:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775960.379589:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775960.379596:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775960.379601:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775960.379603:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775960.379608:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775960.379615:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775960.379618:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775960.379623:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775775960.379628:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775960.379633:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775960.379634:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775960.379639:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775960.379640:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775960.379643:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775960.379649:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775775960.379657:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775960.379666:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710378624/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775960.379686:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775960.379690:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775960.379702:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710378624/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775960.379720:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775775960.379729:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775775960.379739:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775775960.379745:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775775960.379748:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775775960.379755:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775775960.379762:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12842 00000400:00000200:2.0:1775775960.379770:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775960.379781:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775775960.379786:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775775960.379792:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775960.379795:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775960.379802:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775960.379808:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000200:1.1:1775775960.379811:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:2.0:1775775960.379813:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775775960.379816:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000800:00000200:1.1:1775775960.379816:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1775775960.379821:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775775960.379825:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775775960.379830:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775775960.379834:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775775960.379855:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775960.379859:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775775960.379863:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775775960.379866:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775775960.379871:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775775960.379875:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775775960.379880:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775775960.379885:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775775960.379890:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775775960.379897:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775960.379900:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775775960.379905:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775775960.379909:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775960.379915:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775775960.379923:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775775960.379928:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775775960.379932:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775775960.379935:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775960.379939:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775775960.379942:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775775960.379945:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775775960.379950:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775775960.379954:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775775960.379959:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710378752/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775775960.379972:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775775960.379976:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775960.379985:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710378752/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775960.380018:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775775960.380024:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775960.380033:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775960.380040:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775775960.380051:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775775960.380059:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12853) 00000800:00000200:1.1:1775775960.380255:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775960.380257:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775775960.380455:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775960.380457:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00010000:00000001:3.0:1775775960.507232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775960.507236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775960.507241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775960.507242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775960.507245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775960.507246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775960.507324:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775960.507328:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775960.507333:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775961.531349:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775961.531354:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775961.531358:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775961.531359:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775961.531362:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775961.531363:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775961.531450:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775961.531454:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775961.531458:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775962.555272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775962.555280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775962.555286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775962.555287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775962.555291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775962.555293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775962.555352:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775962.555356:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775962.555359:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775963.579283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775963.579293:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775963.579296:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775963.579300:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775963.579305:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775963.579308:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775963.579310:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775963.579312:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775963.579322:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775963.579324:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775963.579328:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775963.579330:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775964.283212:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307519238, at 4307519296 00000100:00000010:0.0:1775775964.283220:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e29c0. 00010000:00000001:3.0:1775775964.603260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775964.603269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775964.603278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775964.603279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775964.603282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775964.603283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775965.499300:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775965.499315:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775965.499319:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775965.499324:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775965.499329:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775965.499344:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12848 00000400:00000200:3.0:1775775965.499353:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775965.499358:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775965.499360:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775965.499364:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775965.499366:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775965.499367:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775965.499369:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775965.499372:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775965.499373:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775965.499375:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775965.499377:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775965.499379:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775965.499381:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775965.499383:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775965.499384:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775965.499385:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775965.499386:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775965.499389:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775965.499392:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775965.499394:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775965.499403:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775965.499406:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775965.499409:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775965.499411:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775965.499414:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775965.499424:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73100. 00000020:00000040:3.0:1775775965.499428:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775965.499432:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775965.499434:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775965.499437:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775965.499440:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775965.499443:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775965.499451:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000001:3.0:1775775965.499458:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775965.499467:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73100 x1862022710378880/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775965.499478:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775965.499483:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775965.499488:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f67e6552] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73100 x1862022710378880/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775965.499511:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775965.499514:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775965.499518:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775965.499520:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775965.499521:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775965.499525:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775965.499527:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12848 00000400:00000200:3.0:1775775965.499533:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775965.499540:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775965.499545:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775965.499550:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775965.499561:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775965.499574:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775965.499582:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775965.499589:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775965.499593:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775965.499598:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775965.499605:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775965.499609:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775965.499615:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775965.499622:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775965.499624:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775965.499625:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775965.499627:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775965.499629:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775965.499634:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775965.499638:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775965.499645:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775965.499650:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775965.499651:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775965.499653:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775965.499656:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775965.499659:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775965.499661:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775965.499665:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775775965.499669:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775965.499673:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775965.499674:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775965.499676:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775965.499681:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775965.499688:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775965.499692:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775775965.499696:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775965.499704:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710379008/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775965.499722:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775965.499725:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775965.499730:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710379008/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775965.499776:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775965.499785:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775965.499793:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775965.499806:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775965.499811:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775965.499820:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775965.499829:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12848 00000400:00000200:3.0:1775775965.499849:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775965.499863:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775965.499869:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775965.499886:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775965.499891:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775965.499897:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775965.499907:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775965.499916:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775965.499923:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775965.499929:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775965.499932:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775965.499940:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775965.499946:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775965.499961:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000800:00000200:1.1:1775775965.499962:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775965.499974:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000100:00000001:3.0:1775775965.499975:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775965.499988:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775965.499998:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775965.500003:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775965.500015:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775965.500019:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775965.500025:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775965.500033:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775965.500036:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775965.500042:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775965.500049:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775965.500063:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775965.500068:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775965.500078:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000020:00000040:3.0:1775775965.500088:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775965.500093:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775965.500096:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775965.500103:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775965.500108:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775965.500111:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775965.500119:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:3.0:1775775965.500125:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775965.500177:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710379136/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775965.500220:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775965.500224:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775965.500235:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710379136/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775965.500303:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775965.500314:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775965.500334:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775965.500340:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775965.500348:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775965.500356:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12858) 00000800:00000200:1.1:1775775965.500604:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775965.500607:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775775965.500822:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775965.500823:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00010000:00000001:3.0:1775775965.627268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775965.627274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775965.627280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775965.627282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775965.627285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775965.627287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775965.627355:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775965.627359:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775965.627363:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775966.651295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775966.651304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775966.651311:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775966.651312:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775966.651317:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775966.651318:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775966.651408:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775966.651414:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775966.651417:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775967.675254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775967.675259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775967.675263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775967.675264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775967.675267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775967.675268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775967.675338:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775967.675341:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775967.675344:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775968.699256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775968.699262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775968.699268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775968.699269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775968.699273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775968.699274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775969.403228:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307524358, at 4307524416 00000100:00000010:0.0:1775775969.403247:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e21e0. 00010000:00000001:3.0:1775775969.723272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775969.723279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775969.723285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775969.723289:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775969.723292:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775969.723295:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775969.723298:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775969.723301:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775969.723303:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775969.723305:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775969.723310:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775969.723312:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775969.723313:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775969.723315:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775969.723316:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775969.723317:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775969.723319:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775969.723321:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775970.619238:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775970.619246:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775970.619250:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775970.619254:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775970.619260:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775970.619267:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12853 00000400:00000200:3.0:1775775970.619281:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775970.619288:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775970.619292:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775970.619295:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775970.619297:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775970.619300:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775970.619302:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775970.619305:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775970.619306:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775970.619309:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775970.619311:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775970.619313:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775970.619315:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775970.619317:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775970.619319:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775970.619320:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775970.619321:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775970.619323:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775970.619325:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775970.619326:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775970.619328:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775970.619330:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775970.619332:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775970.619334:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775970.619335:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775970.619343:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775775970.619345:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775970.619349:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775970.619351:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775970.619353:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775970.619355:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775970.619358:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775970.619365:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000001:3.0:1775775970.619370:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775970.619377:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710379264/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775970.619385:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775970.619389:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775970.619392:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710379264/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775970.619417:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775970.619420:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775970.619423:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775970.619424:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775970.619426:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775970.619430:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775970.619434:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12853 00000400:00000200:3.0:1775775970.619439:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775970.619442:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775970.619447:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775970.619453:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775970.619458:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775970.619464:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775970.619470:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775970.619476:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775970.619480:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775970.619485:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775970.619489:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775970.619493:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775970.619494:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775970.619497:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775970.619498:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775970.619500:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775970.619501:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775970.619507:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775970.619511:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775970.619514:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775970.619516:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775970.619523:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775970.619531:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775970.619534:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775970.619538:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775970.619544:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775970.619546:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775970.619550:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775775970.619553:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775970.619555:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775970.619556:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775970.619558:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775970.619561:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775970.619564:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775970.619570:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000100:00000001:3.0:1775775970.619575:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775970.619584:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710379392/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775970.619594:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775970.619597:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775970.619604:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710379392/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775970.619629:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775970.619633:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775970.619636:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775970.619639:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775970.619640:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775970.619643:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775970.619647:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12853 00000400:00000200:3.0:1775775970.619656:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775970.619675:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775970.619682:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775970.619687:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775970.619692:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775970.619698:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775970.619701:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775970.619703:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775970.619705:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775970.619709:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775970.619711:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775970.619713:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775970.619717:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775970.619720:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775970.619724:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775970.619727:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775970.619729:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775970.619731:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775970.619732:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775970.619734:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775970.619740:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775970.619743:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.1:1775775970.619743:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:3.0:1775775970.619747:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775970.619748:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000100:00000001:3.0:1775775970.619749:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775970.619752:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775970.619756:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775970.619759:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775970.619764:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775775970.619767:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775970.619770:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775970.619773:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775970.619775:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775970.619777:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775970.619779:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775970.619784:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000001:3.0:1775775970.619789:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775970.619796:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710379520/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775970.619825:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775970.619827:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775970.619833:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710379520/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775970.619866:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775970.619872:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775970.619877:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775970.619881:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775970.619888:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775970.619895:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12863) 00000800:00000200:1.1:1775775970.620176:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775970.620177:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775775970.620372:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775970.620375:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00010000:00000001:3.0:1775775970.747243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775970.747247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775970.747252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775970.747253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775970.747264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775970.747265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775970.747335:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775970.747338:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775970.747341:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775971.771284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775971.771291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775971.771297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775971.771298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775971.771303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775971.771304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775971.771378:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775971.771382:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775971.771385:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775972.795241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775972.795260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775972.795270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775972.795272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775972.795276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775972.795278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775972.795343:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775972.795347:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775972.795351:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775973.819260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775973.819268:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775973.819270:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775973.819274:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775973.819279:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775973.819281:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775973.819283:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775973.819286:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775973.819290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775973.819292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775973.819296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775973.819298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775974.523233:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307529478, at 4307529536 00000100:00000010:0.0:1775775974.523250:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2cc0. 00010000:00000001:3.0:1775775974.843267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775974.843276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775974.843284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775974.843285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775974.843289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775974.843290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775975.739206:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775975.739218:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775975.739223:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775975.739229:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775975.739234:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775975.739245:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12858 00000400:00000200:3.0:1775775975.739268:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775975.739280:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775975.739283:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775975.739287:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775975.739291:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775975.739293:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775975.739296:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775975.739300:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775975.739302:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775975.739306:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775975.739309:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775975.739311:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775975.739313:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775975.739315:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775975.739317:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775975.739318:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775975.739319:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775975.739321:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775975.739324:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775975.739325:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775975.739328:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775975.739330:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775975.739332:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775975.739334:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775975.739335:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775975.739343:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775775975.739346:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775975.739349:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775975.739351:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775975.739353:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775975.739356:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775975.739359:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775975.739366:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775775975.739372:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775975.739378:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710379648/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775975.739386:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775975.739389:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775975.739393:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710379648/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775975.739412:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775975.739414:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775975.739417:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775975.739418:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775975.739419:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775975.739421:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775975.739423:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12858 00000400:00000200:3.0:1775775975.739427:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775975.739430:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775975.739432:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775975.739437:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775975.739441:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775975.739444:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775975.739447:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775975.739449:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775975.739452:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775975.739454:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775975.739457:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775975.739461:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775975.739467:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775975.739473:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775975.739476:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775975.739478:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775975.739481:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775975.739484:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775975.739489:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775975.739495:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775975.739497:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775975.739503:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775975.739509:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775975.739512:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775975.739516:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775975.739519:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775975.739523:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775975.739527:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000020:00000040:3.0:1775775975.739531:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775975.739534:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775975.739535:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775975.739541:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775975.739542:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775975.739543:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775975.739554:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000100:00000001:3.0:1775775975.739561:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775975.739571:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710379776/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775975.739594:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775975.739599:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775975.739609:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710379776/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775975.739626:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775975.739631:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775975.739640:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775975.739645:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775975.739648:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775975.739654:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775975.739661:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12858 00000400:00000200:3.0:1775775975.739674:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775975.739681:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775975.739684:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775975.739691:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775975.739695:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775975.739699:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775975.739704:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775975.739709:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775975.739710:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775975.739712:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775975.739714:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775975.739716:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775975.739718:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775975.739725:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775975.739727:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775975.739730:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775975.739734:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775975.739737:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000800:00000200:1.1:1775775975.739738:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00080000:3.0:1775775975.739742:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000800:00000200:1.1:1775775975.739744:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000100:00000001:3.0:1775775975.739745:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775975.739750:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775975.739754:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775975.739758:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775975.739760:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775975.739762:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775975.739765:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775975.739768:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775975.739772:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775775975.739776:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775975.739779:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775975.739781:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775975.739784:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775975.739785:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775975.739787:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775975.739790:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000100:00000001:3.0:1775775975.739794:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775975.739802:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710379904/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775975.739821:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775975.739824:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775975.739832:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710379904/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775975.739852:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775975.739857:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775975.739864:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775975.739868:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775975.739871:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775975.739877:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12868) 00000800:00000200:1.1:1775775975.740104:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775975.740105:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775775975.740307:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775975.740309:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00010000:00000001:3.0:1775775975.867251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775975.867255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775975.867259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775975.867260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775975.867262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775975.867263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775975.867322:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775975.867326:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775975.867330:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775976.891192:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775976.891202:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775976.891208:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775976.891210:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775976.891214:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775976.891215:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775976.891309:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775976.891315:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775976.891320:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775977.915272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775977.915278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775977.915282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775977.915283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775977.915286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775977.915287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775977.915349:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775977.915352:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775977.915355:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775978.939280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775978.939289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775978.939296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775978.939298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775978.939302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775978.939303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775979.643269:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307534598, at 4307534656 00000100:00000010:0.0:1775775979.643291:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2d20. 00010000:00000001:3.0:1775775979.963252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775979.963257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775979.963264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775979.963267:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775979.963269:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775979.963273:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775979.963276:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775979.963278:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775979.963286:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775979.963289:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775979.963294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775979.963295:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775979.963296:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775979.963298:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775979.963299:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775979.963300:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775979.963301:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775979.963302:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775980.859235:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775980.859251:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775775980.859255:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775980.859260:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775980.859266:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775980.859286:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12863 00000400:00000200:3.0:1775775980.859301:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775980.859308:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775980.859312:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775980.859315:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775980.859319:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775980.859320:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775980.859322:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775980.859326:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775980.859327:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775775980.859330:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775980.859332:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775775980.859334:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775775980.859336:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775775980.859337:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775980.859338:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775980.859339:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775980.859340:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775980.859342:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775775980.859345:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775980.859346:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775980.859349:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775980.859352:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775980.859354:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775980.859356:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775775980.859357:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775980.859367:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775775980.859370:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775775980.859374:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775980.859376:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775980.859379:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775980.859382:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775980.859386:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775980.859393:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000100:00000001:3.0:1775775980.859399:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775980.859408:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710380032/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775980.859417:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775980.859422:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775980.859425:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710380032/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775980.859445:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775980.859447:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775980.859450:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775775980.859451:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775980.859452:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775980.859455:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775980.859456:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12863 00000400:00000200:3.0:1775775980.859459:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775980.859461:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775980.859463:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775980.859465:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775980.859467:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775980.859469:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775980.859470:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775980.859472:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775980.859472:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775775980.859473:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775980.859474:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775775980.859477:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775775980.859478:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775775980.859481:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775980.859495:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775980.859500:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775980.859502:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775980.859506:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775775980.859511:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775980.859513:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775980.859514:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775980.859520:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775980.859523:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775980.859528:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775980.859533:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775980.859541:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775980.859546:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775980.859552:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775775980.859558:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775980.859563:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775980.859566:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775980.859568:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775980.859570:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775980.859572:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775980.859585:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000100:00000001:3.0:1775775980.859589:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775980.859599:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710380160/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775980.859611:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775980.859614:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775980.859619:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710380160/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775980.859640:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775775980.859642:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775775980.859645:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775775980.859646:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775775980.859650:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775775980.859654:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775775980.859657:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12863 00000400:00000200:3.0:1775775980.859664:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775775980.859677:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775775980.859681:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775775980.859694:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775980.859697:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775980.859704:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775980.859710:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775775980.859717:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775775980.859720:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775775980.859723:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775775980.859725:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775775980.859730:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775775980.859738:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775775980.859746:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775980.859754:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775775980.859757:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775775980.859764:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775775980.859767:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775775980.859772:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775775980.859776:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775775980.859782:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775775980.859789:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775775980.859792:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775980.859797:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775775980.859799:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775775980.859810:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775980.859812:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775775980.859816:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775775980.859825:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775775980.859827:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775775980.859830:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775775980.859836:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775775980.859837:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775775980.859839:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775775980.859842:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775775980.859849:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775775980.859861:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710380288/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775775980.859874:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775775980.859878:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775980.859886:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710380288/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775775980.859891:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775775980.859895:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775980.859901:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775980.859906:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775775980.859918:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775775980.859927:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12873) 00000800:00000200:1.1:1775775980.859972:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775980.860238:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775980.860338:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00010000:00000001:3.0:1775775980.987297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775980.987306:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775980.987315:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775980.987316:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775980.987320:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775980.987321:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775980.987423:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775980.987429:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775980.987434:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775982.011263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775982.011270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775982.011275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775982.011276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775982.011279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775982.011280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775982.011353:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775982.011357:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775982.011360:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775983.035266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775983.035272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775983.035278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775983.035279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775983.035284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775983.035285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775983.035407:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775983.035412:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775983.035416:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775775983.931466:0:2401:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:2.0:1775775983.931509:0:2401:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1775775983.931519:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775983.931523:0:2401:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:2.0:1775775983.931529:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775775983.931534:0:2401:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:2.0:1775775983.931539:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775983.931544:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:2.0:1775775983.931550:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775983.931555:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775983.931562:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3440 00000400:00000010:2.0:1775775983.931566:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3440. 00000100:00000001:2.0:1775775983.931574:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775983.931586:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c71180 x1862022710380032/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776005 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775983.931614:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775983.931618:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71180 x1862022710380032/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776005 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775983.931625:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775983.931626:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775775983.931637:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775983.931640:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230000. 00000400:00000200:2.0:1775775983.931643:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775983.931647:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775983.931650:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f34c8 00000400:00000010:2.0:1775775983.931653:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f34c8. 00000100:00000001:2.0:1775775983.931658:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775983.931664:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c72a00 x1862022710380160/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776005 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775983.931675:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775983.931681:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72a00 x1862022710380160/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776005 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775983.931692:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775983.931694:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775775983.931702:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775775983.931706:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230c00. 00000400:00000200:2.0:1775775983.931711:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775775983.931717:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775775983.931722:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3000 00000400:00000010:2.0:1775775983.931726:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3000. 00000100:00000001:2.0:1775775983.931729:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775775983.931734:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c72300 x1862022710380288/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776005 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775983.931754:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775775983.931760:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72300 x1862022710380288/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776005 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775775983.931778:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775775983.931781:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:3.0:1775775984.059262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775984.059268:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775984.059270:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775984.059274:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775984.059288:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775984.059291:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775984.059293:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775984.059296:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775984.059300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775984.059302:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775984.059306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775984.059307:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775984.763271:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307539718, at 4307539776 00000100:00000010:0.0:1775775984.763299:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ae0. 00010000:00000001:3.0:1775775985.083256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775985.083259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775985.083266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775985.083268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775985.083272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775985.083273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775985.979271:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775985.979284:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775985.979289:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775985.979296:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775985.979303:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775985.979315:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12868 00000400:00000200:0.0:1775775985.979331:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775985.979338:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775985.979342:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775985.979348:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775985.979352:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775985.979355:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775985.979357:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775985.979361:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775985.979363:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775985.979365:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775985.979368:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775985.979371:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775985.979374:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775985.979377:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775985.979378:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775985.979380:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775985.979382:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775985.979386:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775985.979389:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775985.979392:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775985.979427:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775985.979432:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775985.979435:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775985.979438:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775985.979441:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775985.979449:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061f80. 00000020:00000040:0.0:1775775985.979453:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775985.979457:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775985.979459:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775985.979462:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775985.979465:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775985.979471:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775985.979481:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6c00. 00000100:00000001:0.0:1775775985.979489:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775985.979501:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061f80 x1862022710380416/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775985.979512:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775985.979518:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775985.979523:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000029d3794] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061f80 x1862022710380416/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775985.979548:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775985.979552:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775985.979555:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775985.979558:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775985.979560:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775985.979564:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775985.979567:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12868 00000400:00000200:0.0:1775775985.979573:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775985.979577:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775985.979580:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775985.979583:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775985.979586:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775985.979591:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775985.979597:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775985.979602:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775985.979603:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775985.979605:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775985.979609:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775985.979615:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775985.979625:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775985.979644:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775985.979649:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775985.979656:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775985.979664:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775985.979669:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775985.979678:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775985.979683:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775985.979693:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775985.979702:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775985.979709:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775985.979713:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775985.979720:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775985.979728:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775985.979732:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775985.979737:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063b80. 00000020:00000040:0.0:1775775985.979745:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775985.979754:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775985.979757:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775985.979759:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775985.979761:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775985.979764:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775985.979770:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7800. 00000100:00000001:0.0:1775775985.979777:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775985.979786:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063b80 x1862022710380544/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775985.979821:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775985.979826:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775985.979837:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ee9007b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063b80 x1862022710380544/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775985.979880:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775985.979889:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775985.979895:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775985.979899:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775985.979904:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775985.979912:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775985.979918:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12868 00000400:00000200:0.0:1775775985.979926:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775985.979937:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775985.979940:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775985.979947:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775985.979957:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775985.979962:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775985.979971:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775985.979978:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775985.979982:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775985.979987:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775985.979993:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775985.979999:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775985.980009:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775985.980023:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775985.980032:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775985.980041:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775985.980047:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775985.980053:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775985.980069:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775985.980075:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775985.980079:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775985.980086:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775985.980090:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775985.980093:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775985.980097:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775985.980101:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775985.980103:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775985.980107:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061500. 00000020:00000040:0.0:1775775985.980109:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775985.980113:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775985.980116:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775985.980124:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775985.980127:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775985.980131:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775775985.980204:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775985.980209:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 02000000:00000010:0.0:1775775985.980212:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5800. 00000100:00000001:0.0:1775775985.980220:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775985.980235:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061500 x1862022710380672/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775985.980251:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775985.980256:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775985.980268:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb8d992] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061500 x1862022710380672/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775985.980288:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775985.980294:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775985.980305:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775985.980313:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775985.980328:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775985.980339:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12878) 00000800:00000200:1.1:1775775985.980783:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775985.980786:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775775985.981112:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775985.981114:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00010000:00000001:3.0:1775775986.107219:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775986.107223:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775986.107227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775986.107228:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775986.107231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775986.107232:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775986.107309:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775986.107316:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775986.107320:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775987.131267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775987.131273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775987.131278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775987.131280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775987.131284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775987.131285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775987.131342:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775987.131345:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775987.131348:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775988.155268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775988.155274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775988.155281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775988.155283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775988.155287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775988.155288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775988.155354:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775988.155371:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775988.155375:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775989.179243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775989.179248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775989.179252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775989.179253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775989.179256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775989.179257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775989.883254:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307544839, at 4307544896 00000100:00000010:0.0:1775775989.883268:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2c60. 00000800:00000200:2.0:1775775989.883389:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775775989.883394:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775775989.883397:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1775775989.883566:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775775989.883583:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775775989.883588:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775775989.883591:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1775775990.203234:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775990.203238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775990.203242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775990.203245:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775990.203247:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775990.203250:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775990.203252:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775990.203254:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775990.203255:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775990.203257:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775990.203260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775990.203261:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775990.203262:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775990.203263:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775990.203263:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775990.203264:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775990.203265:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775990.203266:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775991.099222:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775991.099232:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775991.099235:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775991.099240:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775991.099246:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775991.099254:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12873 00000400:00000200:0.0:1775775991.099264:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775991.099269:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775991.099272:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775991.099275:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775991.099278:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775991.099280:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775991.099283:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775991.099286:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775991.099288:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775991.099290:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775991.099292:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775991.099294:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775991.099296:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775991.099298:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775991.099299:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775991.099300:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775991.099301:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775991.099303:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775991.099321:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775991.099323:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775991.099325:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775991.099327:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775991.099332:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775991.099334:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775991.099335:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775991.099343:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062a00. 00000020:00000040:0.0:1775775991.099346:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775991.099349:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775991.099351:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775991.099353:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775991.099354:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775991.099358:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775991.099364:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5c00. 00000100:00000001:0.0:1775775991.099369:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775991.099377:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062a00 x1862022710380800/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775991.099384:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775991.099388:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775991.099391:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009753f9f6] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062a00 x1862022710380800/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775991.099412:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775991.099414:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775991.099417:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775991.099418:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775991.099419:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775991.099421:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775991.099423:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12873 00000400:00000200:0.0:1775775991.099426:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775991.099430:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775991.099434:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775991.099437:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775991.099441:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775991.099446:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775991.099449:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775991.099454:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775991.099457:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775991.099461:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775991.099465:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775991.099470:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775991.099475:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775991.099482:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775991.099487:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775991.099492:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775991.099493:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775991.099498:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775991.099502:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775991.099505:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775991.099507:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775991.099513:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775991.099517:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775991.099518:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775991.099523:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775991.099525:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775991.099529:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775991.099533:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062d80. 00000020:00000040:0.0:1775775991.099536:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775991.099538:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775991.099540:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775991.099544:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775991.099545:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775991.099547:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775991.099551:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7c00. 00000100:00000001:0.0:1775775991.099555:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775991.099562:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062d80 x1862022710380928/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775991.099573:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775991.099576:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775991.099584:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009965f80b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062d80 x1862022710380928/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775991.099602:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775991.099606:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775991.099609:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775991.099612:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775991.099614:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775991.099618:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775991.099622:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12873 00000400:00000200:0.0:1775775991.099628:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775991.099632:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775991.099635:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775991.099639:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775991.099643:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775991.099647:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775991.099649:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775991.099651:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775991.099654:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775991.099657:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775991.099659:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775991.099661:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775991.099664:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775991.099669:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775991.099671:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775991.099674:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775991.099678:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775991.099682:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775991.099687:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775991.099692:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775991.099694:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775991.099696:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775991.099699:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775991.099701:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775991.099704:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775991.099710:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775991.099714:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775991.099719:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062680. 00000020:00000040:0.0:1775775991.099722:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775991.099726:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775991.099729:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775991.099733:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775991.099737:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775991.099740:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775991.099745:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4800. 00000100:00000001:0.0:1775775991.099751:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775991.099758:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062680 x1862022710381056/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775991.099770:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775991.099774:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775991.099781:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000082c7985e] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062680 x1862022710381056/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1775775991.099782:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775991.099787:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000100:00000001:0.0:1775775991.099797:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775991.099800:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775991.099803:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775991.099806:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775991.099811:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775991.099818:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12883) 00000800:00000200:1.1:1775775991.100115:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775991.100118:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775775991.100454:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775991.100457:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00010000:00000001:3.0:1775775991.227226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775991.227232:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775991.227237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775991.227238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775991.227242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775991.227243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775991.227322:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775991.227327:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775991.227330:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775992.251261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775992.251267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775992.251272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775992.251274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775992.251278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775992.251280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775992.251371:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775992.251375:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775992.251378:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775993.275261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775993.275265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775993.275291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775993.275292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775993.275295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775993.275296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775993.275354:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775993.275357:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775993.275359:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775994.299263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775994.299271:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775994.299274:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775994.299278:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775994.299283:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775994.299285:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775994.299286:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775994.299288:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775994.299291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775994.299292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775994.299295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775994.299296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775995.003325:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307549958, at 4307550016 00000100:00000010:0.0:1775775995.003333:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2540. 00010000:00000001:3.0:1775775995.323299:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775995.323305:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775995.323312:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775995.323314:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775995.323318:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775995.323320:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775996.219287:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775996.219294:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775775996.219297:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775996.219301:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775996.219305:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775996.219313:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12878 00000400:00000200:0.0:1775775996.219323:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775996.219329:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775996.219332:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775996.219336:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775996.219340:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775996.219342:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775996.219346:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775996.219350:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775996.219352:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775775996.219355:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775996.219357:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775775996.219360:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775775996.219362:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775775996.219366:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775996.219367:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775996.219369:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775996.219372:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775996.219375:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775775996.219378:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775996.219381:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775996.219385:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775996.219388:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775996.219397:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775996.219400:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775775996.219402:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775996.219412:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060380. 00000020:00000040:0.0:1775775996.219416:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775775996.219421:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775996.219423:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775996.219426:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775996.219429:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775996.219433:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775996.219440:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7c00. 00000100:00000001:0.0:1775775996.219445:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775996.219453:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060380 x1862022710381184/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775996.219460:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775996.219464:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775996.219468:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000614ace82] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060380 x1862022710381184/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775996.219485:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775996.219487:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775996.219490:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775775996.219491:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775996.219492:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775996.219494:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775996.219496:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12878 00000400:00000200:0.0:1775775996.219499:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775996.219502:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775996.219504:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775996.219505:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775996.219507:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775996.219508:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775996.219510:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775996.219513:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775996.219514:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775775996.219517:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775996.219519:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775775996.219522:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775775996.219524:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775775996.219528:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775996.219532:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775996.219536:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775996.219541:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775996.219545:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775775996.219552:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775996.219558:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775996.219564:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775996.219577:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775996.219582:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775996.219585:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775996.219590:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775996.219593:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775996.219597:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775996.219602:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063800. 00000020:00000040:0.0:1775775996.219605:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775996.219608:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775996.219609:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775996.219610:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775996.219612:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775996.219613:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775996.219617:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5c00. 00000100:00000001:0.0:1775775996.219619:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775996.219624:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063800 x1862022710381312/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775996.219633:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775996.219636:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775996.219640:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000015d81d25] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063800 x1862022710381312/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775996.219647:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775775996.219653:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775775996.219658:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775775996.219659:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775775996.219660:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775775996.219664:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775775996.219667:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12878 00000400:00000200:0.0:1775775996.219672:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775775996.219677:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775775996.219680:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775775996.219683:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775996.219688:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775996.219691:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775996.219692:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775775996.219694:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775775996.219695:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775775996.219698:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775775996.219701:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775775996.219704:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775775996.219707:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775775996.219713:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775996.219717:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775775996.219722:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775775996.219728:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775775996.219735:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775775996.219742:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775775996.219748:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775775996.219753:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775775996.219758:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775775996.219763:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775996.219769:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775775996.219772:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775775996.219779:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775996.219782:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775775996.219789:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062680. 00000020:00000040:0.0:1775775996.219794:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775775996.219799:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775775996.219801:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775775996.219805:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775775996.219808:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775775996.219813:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775775996.219819:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7000. 00000100:00000001:0.0:1775775996.219827:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775775996.219836:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062680 x1862022710381440/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775775996.219857:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775775996.219860:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775996.219868:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000082c7985e] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062680 x1862022710381440/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775775996.219883:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775775996.219887:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775996.219892:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775775996.219896:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775775996.219900:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775775996.219904:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:0.0:1775775996.219907:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12888) 00000800:00000200:1.1:1775775996.219910:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775775996.220390:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775996.220393:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775775996.220704:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775775996.220706:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00010000:00000001:3.0:1775775996.347238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775996.347244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775996.347251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775996.347253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775996.347257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775996.347258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775996.347314:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775996.347318:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775996.347320:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775997.371261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775997.371266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775997.371271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775997.371273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775997.371277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775997.371279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775997.371329:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775997.371333:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775997.371336:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775998.395250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775998.395256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775998.395262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775998.395263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775998.395267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775998.395268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775775998.395319:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775775998.395323:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775775998.395325:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775775999.419233:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775999.419240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775999.419245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775999.419248:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775999.419250:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775999.419252:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775999.419255:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775999.419256:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775999.419258:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775999.419259:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775999.419262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775775999.419263:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775775999.419264:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775775999.419265:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775999.419266:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775775999.419267:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775999.419268:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775775999.419269:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776000.123295:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307555078, at 4307555136 00000100:00000010:0.0:1775776000.123311:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2c60. 00010000:00000001:3.0:1775776000.443209:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776000.443214:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776000.443218:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776000.443219:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776000.443222:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776000.443223:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776001.339307:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776001.339315:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776001.339319:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776001.339322:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776001.339326:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776001.339335:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12883 00000400:00000200:2.0:1775776001.339348:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776001.339356:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776001.339358:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776001.339362:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776001.339365:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776001.339367:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776001.339369:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776001.339374:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776001.339376:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776001.339379:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776001.339381:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776001.339383:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776001.339385:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776001.339387:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776001.339389:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776001.339390:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776001.339391:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776001.339394:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776001.339396:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776001.339398:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776001.339401:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776001.339404:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776001.339406:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776001.339407:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776001.339409:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776001.339426:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775776001.339429:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776001.339432:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776001.339433:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776001.339436:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776001.339438:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776001.339443:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776001.339451:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775776001.339458:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776001.339467:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710381568/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776001.339476:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776001.339481:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776001.339484:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710381568/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776001.339509:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776001.339512:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776001.339515:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776001.339516:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776001.339517:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776001.339519:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776001.339520:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12883 00000400:00000200:2.0:1775776001.339523:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776001.339525:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776001.339527:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776001.339530:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776001.339532:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776001.339534:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776001.339536:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776001.339537:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776001.339538:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776001.339540:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776001.339542:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776001.339545:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776001.339547:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776001.339552:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776001.339555:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776001.339558:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776001.339560:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776001.339561:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776001.339563:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776001.339564:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776001.339567:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776001.339574:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776001.339577:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776001.339580:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776001.339586:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776001.339591:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776001.339594:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776001.339601:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775776001.339605:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776001.339607:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776001.339611:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776001.339612:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776001.339613:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776001.339615:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776001.339620:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775776001.339626:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776001.339632:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710381696/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776001.339642:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776001.339646:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776001.339649:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710381696/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776001.339661:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776001.339664:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776001.339669:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776001.339670:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776001.339671:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776001.339673:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776001.339675:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12883 00000400:00000200:2.0:1775776001.339679:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776001.339683:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776001.339684:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776001.339686:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776001.339688:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776001.339692:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776001.339695:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776001.339699:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776001.339701:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776001.339703:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776001.339705:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776001.339708:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776001.339717:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776001.339723:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776001.339726:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776001.339728:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776001.339731:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776001.339733:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776001.339736:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776001.339739:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776001.339741:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776001.339744:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776001.339746:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776001.339747:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776001.339749:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776001.339753:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776001.339756:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776001.339759:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775776001.339762:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776001.339765:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776001.339766:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776001.339767:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776001.339769:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776001.339772:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776001.339775:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775776001.339779:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776001.339791:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710381824/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776001.339805:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776001.339808:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776001.339815:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710381824/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776001.339829:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776001.339833:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776001.339837:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776001.339839:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776001.339842:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776001.339846:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12894) 00000800:00000200:1.1:1775776001.339856:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776001.339861:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775776001.340301:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776001.340303:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775776001.340510:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776001.340512:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00010000:00000001:3.0:1775776001.467243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776001.467249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776001.467255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776001.467256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776001.467260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776001.467261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776001.467346:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776001.467351:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776001.467355:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776002.491271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776002.491278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776002.491284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776002.491285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776002.491289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776002.491290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776002.491362:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776002.491367:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776002.491370:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776003.515238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776003.515246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776003.515253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776003.515254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776003.515258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776003.515259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776003.515331:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776003.515336:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776003.515341:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776004.539265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776004.539274:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776004.539276:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776004.539278:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776004.539282:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776004.539284:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776004.539285:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776004.539287:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776004.539290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776004.539291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776004.539294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776004.539295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776005.243274:0:214567:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307560198, at 4307560256 00000100:00000010:0.0:1775776005.243292:0:214567:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ae0. 00010000:00000001:3.0:1775776005.563277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776005.563286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776005.563293:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776005.563294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776005.563306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776005.563308:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776006.459329:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776006.459340:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776006.459343:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776006.459349:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776006.459354:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776006.459363:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12889 00000400:00000200:2.0:1775776006.459373:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776006.459382:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776006.459386:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776006.459390:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776006.459393:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776006.459396:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776006.459399:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776006.459404:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776006.459406:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776006.459409:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776006.459411:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776006.459414:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776006.459416:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776006.459420:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776006.459422:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776006.459424:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776006.459426:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776006.459430:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776006.459433:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776006.459436:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776006.459440:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776006.459444:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776006.459447:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776006.459450:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776006.459453:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776006.459463:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775776006.459467:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776006.459471:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776006.459473:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776006.459476:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776006.459479:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776006.459483:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776006.459492:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775776006.459500:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776006.459509:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710381952/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776006.459521:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776006.459526:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776006.459531:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710381952/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776006.459554:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776006.459558:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776006.459562:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776006.459564:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776006.459566:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776006.459570:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776006.459574:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12889 00000400:00000200:2.0:1775776006.459580:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776006.459584:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776006.459588:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776006.459594:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776006.459601:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776006.459605:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776006.459609:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776006.459617:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776006.459620:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776006.459622:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776006.459624:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776006.459632:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776006.459642:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776006.459650:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776006.459657:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776006.459661:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776006.459666:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776006.459677:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776006.459688:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776006.459697:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776006.459702:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776006.459710:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776006.459717:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776006.459725:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776006.459731:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776006.459737:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776006.459741:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776006.459752:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775776006.459757:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776006.459765:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776006.459769:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776006.459773:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776006.459775:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776006.459778:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776006.459783:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775776006.459789:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776006.459798:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710382080/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776006.459821:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776006.459824:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776006.459831:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710382080/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776006.459869:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776006.459879:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776006.459890:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776006.459892:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776006.459897:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776006.459908:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776006.459912:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12889 00000400:00000200:2.0:1775776006.459920:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776006.459930:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776006.459935:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776006.459943:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776006.459948:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776006.459955:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776006.459962:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776006.459969:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776006.459973:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776006.459977:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776006.459985:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776006.459996:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776006.460004:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776006.460012:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776006.460014:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776006.460016:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776006.460018:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776006.460021:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776006.460029:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776006.460051:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776006.460057:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776006.460063:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776006.460069:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776006.460073:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776006.460075:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776006.460080:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776006.460084:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776006.460089:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775776006.460097:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776006.460103:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776006.460107:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776006.460110:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.1:1775776006.460113:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:2.0:1775776006.460114:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776006.460119:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776006.460122:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 02000000:00000010:2.0:1775776006.460125:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775776006.460173:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776006.460185:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710382208/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776006.460199:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776006.460205:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776006.460213:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710382208/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776006.460249:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776006.460256:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776006.460262:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776006.460267:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776006.460285:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776006.460297:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12899) 00000800:00000200:1.1:1775776006.460724:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776006.460727:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775776006.461057:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776006.461060:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00010000:00000001:3.0:1775776006.587279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776006.587285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776006.587292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776006.587293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776006.587296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776006.587296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776006.587367:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776006.587372:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776006.587375:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776007.611260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776007.611264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776007.611268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776007.611270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776007.611273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776007.611274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776007.611335:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776007.611338:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776007.611340:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776008.635295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776008.635304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776008.635313:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776008.635314:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776008.635318:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776008.635320:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776008.635433:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776008.635439:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776008.635444:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776009.659281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776009.659288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776009.659293:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776009.659297:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776009.659299:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776009.659302:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776009.659305:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776009.659307:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776009.659308:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776009.659310:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776009.659313:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776009.659314:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776009.659315:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776009.659316:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776009.659317:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776009.659318:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776009.659319:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776009.659320:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776010.363235:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307565319, at 4307565376 00000100:00000010:0.0:1775776010.363248:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2d20. 00010000:00000001:3.0:1775776010.683227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776010.683233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776010.683239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776010.683240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776010.683243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776010.683244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776011.579280:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776011.579291:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776011.579294:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776011.579301:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776011.579307:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776011.579318:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12894 00000400:00000200:0.0:1775776011.579330:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776011.579336:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776011.579341:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776011.579343:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776011.579346:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776011.579348:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776011.579352:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776011.579357:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776011.579359:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776011.579363:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776011.579366:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776011.579369:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776011.579372:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776011.579376:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776011.579378:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776011.579380:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776011.579383:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776011.579386:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776011.579390:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776011.579393:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776011.579397:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776011.579402:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776011.579405:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776011.579408:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776011.579410:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776011.579422:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062d80. 00000020:00000040:0.0:1775776011.579427:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776011.579432:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776011.579435:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776011.579438:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776011.579441:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776011.579446:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776011.579455:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5800. 00000100:00000001:0.0:1775776011.579463:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776011.579475:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062d80 x1862022710382336/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776011.579488:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776011.579566:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776011.579571:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009965f80b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062d80 x1862022710382336/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776011.579673:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776011.579679:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776011.579708:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776011.579712:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776011.579716:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776011.579740:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776011.579747:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12894 00000400:00000200:0.0:1775776011.579769:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776011.579772:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776011.579774:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776011.579778:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776011.579787:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776011.579790:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776011.579793:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776011.579796:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776011.579798:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776011.579809:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776011.579812:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775776011.579816:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775776011.579820:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775776011.579833:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776011.579834:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776011.579839:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776011.579843:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776011.579846:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775776011.579848:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776011.579850:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776011.579851:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776011.579863:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776011.579868:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776011.579870:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776011.579874:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776011.579878:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776011.579887:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776011.579893:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062a00. 00000020:00000040:0.0:1775776011.579896:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776011.579900:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776011.579903:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776011.579907:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776011.579909:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776011.579925:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776011.579931:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7800. 00000100:00000001:0.0:1775776011.579965:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776011.579975:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062a00 x1862022710382464/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1775776011.579998:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776011.580006:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000100:00000001:0.0:1775776011.580008:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776011.580011:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776011.580025:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009753f9f6] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062a00 x1862022710382464/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776011.580036:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776011.580041:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776011.580055:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776011.580057:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776011.580059:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776011.580062:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776011.580067:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12894 00000400:00000200:0.0:1775776011.580079:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776011.580085:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776011.580087:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776011.580090:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776011.580094:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776011.580109:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776011.580112:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776011.580115:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776011.580117:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776011.580118:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776011.580121:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775776011.580125:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775776011.580199:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775776011.580206:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776011.580209:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776011.580222:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776011.580228:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776011.580233:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775776011.580236:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776011.580240:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776011.580243:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776011.580246:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776011.580249:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776011.580254:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776011.580257:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776011.580260:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776011.580263:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776011.580268:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061500. 00000020:00000040:0.0:1775776011.580281:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776011.580286:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776011.580288:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776011.580292:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776011.580295:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776011.580298:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776011.580302:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6c00. 00000100:00000001:0.0:1775776011.580316:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776011.580323:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061500 x1862022710382592/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776011.580348:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776011.580352:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776011.580358:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb8d992] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061500 x1862022710382592/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776011.580379:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776011.580402:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776011.580408:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776011.580413:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776011.580418:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776011.580422:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12904) 00000800:00000200:1.1:1775776011.580643:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776011.580645:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775776011.580936:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776011.580938:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00010000:00000001:3.0:1775776011.707281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776011.707289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776011.707296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776011.707298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776011.707302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776011.707303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776011.707362:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776011.707368:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776011.707372:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776012.731263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776012.731268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776012.731273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776012.731274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776012.731277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776012.731278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776012.731354:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776012.731358:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776012.731362:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776013.755289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776013.755296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776013.755303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776013.755304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776013.755308:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776013.755309:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776013.755402:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776013.755406:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776013.755409:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776014.779260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776014.779267:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776014.779269:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776014.779272:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776014.779276:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776014.779279:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776014.779281:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776014.779283:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776014.779287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776014.779288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776014.779292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776014.779293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776015.483226:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307570439, at 4307570496 00000100:00000010:0.0:1775776015.483241:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2cc0. 00010000:00000001:3.0:1775776015.803214:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776015.803219:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776015.803224:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776015.803225:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776015.803229:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776015.803231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776016.699262:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776016.699271:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776016.699274:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776016.699278:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776016.699282:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776016.699290:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12899 00000400:00000200:2.0:1775776016.699309:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776016.699318:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776016.699321:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776016.699325:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776016.699328:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776016.699331:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776016.699333:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776016.699336:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776016.699338:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776016.699341:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776016.699344:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776016.699346:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776016.699349:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776016.699351:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776016.699353:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776016.699355:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776016.699356:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776016.699359:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776016.699363:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776016.699365:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776016.699368:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776016.699371:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776016.699374:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776016.699376:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776016.699378:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776016.699386:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775776016.699389:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776016.699392:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776016.699394:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776016.699396:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776016.699399:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776016.699403:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776016.699409:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775776016.699416:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776016.699424:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710382720/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776016.699433:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776016.699444:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776016.699448:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710382720/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776016.699465:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776016.699468:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776016.699472:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776016.699473:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776016.699474:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776016.699477:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776016.699480:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12899 00000400:00000200:2.0:1775776016.699484:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776016.699488:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776016.699492:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776016.699497:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776016.699500:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776016.699504:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776016.699506:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776016.699511:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776016.699514:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776016.699517:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776016.699521:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776016.699526:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776016.699529:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776016.699535:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776016.699539:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776016.699545:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776016.699549:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776016.699553:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776016.699558:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776016.699561:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776016.699565:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776016.699571:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776016.699575:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776016.699578:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776016.699582:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776016.699586:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776016.699588:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776016.699594:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775776016.699598:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776016.699602:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776016.699605:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776016.699609:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776016.699611:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776016.699615:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776016.699621:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775776016.699628:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776016.699635:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710382848/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776016.699649:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776016.699653:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776016.699658:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710382848/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776016.699670:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776016.699674:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776016.699679:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776016.699681:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776016.699684:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776016.699690:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776016.699695:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12899 00000400:00000200:2.0:1775776016.699704:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776016.699711:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776016.699714:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776016.699718:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776016.699723:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776016.699726:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776016.699728:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776016.699730:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776016.699732:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776016.699733:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776016.699735:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776016.699737:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776016.699739:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776016.699742:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776016.699744:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776016.699749:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776016.699753:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776016.699758:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776016.699764:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776016.699768:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776016.699771:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776016.699774:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776016.699776:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776016.699778:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776016.699779:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776016.699782:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776016.699785:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776016.699789:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775776016.699792:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776016.699796:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776016.699798:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776016.699802:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776016.699804:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:1.1:1775776016.699806:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:2.0:1775776016.699807:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776016.699813:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000800:00000200:1.1:1775776016.699813:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000100:00000001:2.0:1775776016.699819:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776016.699827:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710382976/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776016.699838:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776016.699842:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776016.699850:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710382976/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776016.699875:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776016.699879:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776016.699884:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776016.699888:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776016.699896:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776016.699903:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12909) 00000800:00000200:1.1:1775776016.700192:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776016.700193:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775776016.700419:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776016.700421:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00010000:00000001:3.0:1775776016.827247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776016.827253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776016.827257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776016.827259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776016.827262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776016.827263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776016.827329:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776016.827332:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776016.827336:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776017.851203:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776017.851210:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776017.851217:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776017.851218:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776017.851222:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776017.851224:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776017.851295:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776017.851300:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776017.851303:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776018.875246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776018.875251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776018.875255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776018.875256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776018.875259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776018.875260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776018.875312:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776018.875315:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776018.875317:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776019.899303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776019.899311:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776019.899318:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776019.899323:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776019.899325:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776019.899330:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776019.899333:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776019.899336:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776019.899338:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776019.899340:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776019.899344:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776019.899346:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776019.899347:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776019.899349:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776019.899350:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776019.899351:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776019.899352:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776019.899354:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776020.603228:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307575558, at 4307575616 00000100:00000010:0.0:1775776020.603243:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e21e0. 00010000:00000001:3.0:1775776020.923229:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776020.923234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776020.923239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776020.923240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776020.923243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776020.923244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776021.819293:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776021.819303:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776021.819308:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776021.819313:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776021.819320:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776021.819331:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12904 00000400:00000200:0.0:1775776021.819353:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776021.819362:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776021.819366:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776021.819369:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776021.819373:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776021.819375:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776021.819378:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776021.819383:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776021.819385:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776021.819389:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776021.819392:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776021.819395:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776021.819398:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776021.819402:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776021.819404:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776021.819406:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776021.819408:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776021.819421:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776021.819430:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776021.819432:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776021.819436:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776021.819440:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776021.819443:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776021.819445:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776021.819447:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776021.819457:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063b80. 00000020:00000040:0.0:1775776021.819461:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776021.819465:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776021.819467:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776021.819471:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776021.819474:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776021.819479:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776021.819489:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000001:0.0:1775776021.819496:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776021.819508:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063b80 x1862022710383104/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776021.819520:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776021.819535:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776021.819541:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ee9007b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063b80 x1862022710383104/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776021.819565:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776021.819569:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776021.819572:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776021.819574:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776021.819582:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776021.819586:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776021.819590:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12904 00000400:00000200:0.0:1775776021.819596:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776021.819603:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776021.819610:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776021.819615:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776021.819618:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776021.819622:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776021.819627:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776021.819640:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776021.819642:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776021.819643:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776021.819646:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775776021.819650:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775776021.819659:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775776021.819665:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776021.819671:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776021.819679:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776021.819685:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776021.819693:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775776021.819703:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776021.819712:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776021.819716:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776021.819723:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776021.819728:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776021.819739:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776021.819747:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776021.819762:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776021.819768:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776021.819788:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061f80. 00000020:00000040:0.0:1775776021.819797:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776021.819807:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776021.819810:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776021.819815:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776021.819818:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776021.819820:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776021.819828:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4400. 00000100:00000001:0.0:1775776021.819838:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776021.819850:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061f80 x1862022710383232/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776021.819870:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776021.819877:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776021.819885:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000029d3794] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061f80 x1862022710383232/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776021.819907:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776021.819913:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776021.819918:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776021.819921:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776021.819923:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776021.819927:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776021.819932:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12904 00000400:00000200:0.0:1775776021.819961:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776021.819967:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776021.819969:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776021.819973:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776021.819977:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776021.819978:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776021.819984:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776021.819988:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776021.819991:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776021.819995:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776021.819997:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775776021.819998:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775776021.820001:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775776021.820006:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776021.820009:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776021.820010:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776021.820011:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776021.820015:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775776021.820017:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776021.820019:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776021.820020:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776021.820022:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.1:1775776021.820025:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:0.0:1775776021.820026:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776021.820033:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776021.820038:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000200:1.1:1775776021.820039:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000008:00000001:0.0:1775776021.820044:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776021.820050:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776021.820055:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061180. 00000020:00000040:0.0:1775776021.820060:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776021.820064:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776021.820066:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776021.820072:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776021.820073:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776021.820075:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776021.820078:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6400. 00000100:00000001:0.0:1775776021.820083:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776021.820091:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061180 x1862022710383360/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776021.820101:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776021.820104:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776021.820113:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e23859] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061180 x1862022710383360/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776021.820130:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776021.820206:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776021.820212:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776021.820220:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776021.820224:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776021.820234:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12914) 00000800:00000200:1.1:1775776021.820575:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776021.820578:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775776021.820867:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776021.820870:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00010000:00000001:3.0:1775776021.947301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776021.947310:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776021.947315:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776021.947316:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776021.947321:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776021.947322:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776021.947389:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776021.947393:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776021.947398:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776022.971188:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776022.971192:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776022.971195:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776022.971196:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776022.971198:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776022.971199:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776022.971260:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776022.971264:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776022.971267:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776023.995275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776023.995280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776023.995286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776023.995287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776023.995291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776023.995293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776023.995322:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776023.995326:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776023.995329:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776025.019241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776025.019247:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776025.019249:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776025.019252:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776025.019256:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776025.019258:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776025.019260:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776025.019262:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776025.019266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776025.019267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776025.019270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776025.019271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776025.723212:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307580679, at 4307580736 00000100:00000010:0.0:1775776025.723225:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e29c0. 00010000:00000001:3.0:1775776026.043258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776026.043263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776026.043268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776026.043270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776026.043273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776026.043275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776026.939223:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776026.939232:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776026.939235:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776026.939238:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776026.939242:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776026.939248:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12909 00000400:00000200:0.0:1775776026.939257:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776026.939262:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776026.939265:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776026.939268:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776026.939270:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776026.939271:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776026.939273:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776026.939276:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776026.939277:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776026.939279:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776026.939281:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776026.939284:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776026.939286:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776026.939289:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776026.939291:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776026.939293:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776026.939295:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776026.939297:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776026.939300:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776026.939303:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776026.939306:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776026.939309:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776026.939312:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776026.939314:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776026.939316:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776026.939325:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060700. 00000020:00000040:0.0:1775776026.939328:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776026.939332:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776026.939334:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776026.939336:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776026.939339:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776026.939342:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776026.939349:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4400. 00000100:00000001:0.0:1775776026.939354:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776026.939361:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060700 x1862022710383488/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776026.939367:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776026.939370:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776026.939373:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000225c8695] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060700 x1862022710383488/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776026.939389:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776026.939391:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776026.939393:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776026.939395:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776026.939396:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776026.939399:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776026.939401:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12909 00000400:00000200:0.0:1775776026.939405:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776026.939407:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776026.939410:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776026.939413:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776026.939416:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776026.939418:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776026.939420:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776026.939423:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776026.939424:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776026.939427:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776026.939430:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775776026.939434:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775776026.939439:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775776026.939444:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776026.939449:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776026.939453:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776026.939456:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776026.939458:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775776026.939463:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776026.939470:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776026.939472:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776026.939479:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776026.939484:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776026.939489:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776026.939495:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776026.939500:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776026.939504:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776026.939509:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061880. 00000020:00000040:0.0:1775776026.939520:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776026.939525:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776026.939527:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776026.939530:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776026.939531:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776026.939533:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776026.939538:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000001:0.0:1775776026.939542:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776026.939548:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061880 x1862022710383616/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776026.939556:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776026.939559:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776026.939564:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005afd1f27] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061880 x1862022710383616/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776026.939576:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776026.939579:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776026.939585:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776026.939587:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776026.939589:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776026.939594:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776026.939598:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12909 00000400:00000200:0.0:1775776026.939603:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776026.939608:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776026.939612:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776026.939615:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776026.939618:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776026.939620:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776026.939623:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776026.939624:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776026.939625:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776026.939626:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776026.939628:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775776026.939631:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775776026.939634:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775776026.939637:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776026.939638:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776026.939640:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776026.939643:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776026.939647:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775776026.939649:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776026.939650:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776026.939653:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776026.939656:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776026.939659:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776026.939662:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776026.939665:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776026.939668:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776026.939670:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776026.939675:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061c00. 00000020:00000040:0.0:1775776026.939679:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776026.939692:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776026.939693:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776026.939697:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776026.939698:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776026.939701:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776026.939705:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6000. 00000100:00000001:0.0:1775776026.939708:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776026.939716:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061c00 x1862022710383744/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776026.939734:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776026.939738:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776026.939745:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076a44879] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061c00 x1862022710383744/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1775776026.939758:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776026.939762:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00000001:0.0:1775776026.939764:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776026.939770:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776026.939776:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776026.939781:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776026.939788:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776026.939795:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12919) 00000800:00000200:1.1:1775776026.940051:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776026.940054:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775776026.940342:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776026.940344:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00010000:00000001:3.0:1775776027.067205:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776027.067210:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776027.067214:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776027.067214:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776027.067217:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776027.067217:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776027.067261:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776027.067265:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776027.067268:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.0:1775776027.771205:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775776027.771212:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775776027.771216:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1775776027.771352:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776027.771367:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775776027.771372:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776027.771373:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1775776028.091219:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776028.091223:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776028.091227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776028.091228:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776028.091230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776028.091231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776028.091290:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776028.091293:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776028.091297:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776029.115236:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776029.115249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776029.115258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776029.115260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776029.115263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776029.115264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776029.115329:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776029.115333:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776029.115336:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776030.139240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776030.139244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776030.139248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776030.139250:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776030.139252:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776030.139257:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776030.139260:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776030.139261:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776030.139263:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776030.139264:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776030.139267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776030.139268:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776030.139269:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776030.139269:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776030.139270:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776030.139271:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776030.139273:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776030.139274:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776030.843250:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307585798, at 4307585856 00000100:00000010:0.0:1775776030.843271:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e22a0. 00010000:00000001:3.0:1775776031.163250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776031.163255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776031.163258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776031.163259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776031.163262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776031.163263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776032.059236:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776032.059241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776032.059247:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1775776032.059247:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776032.059250:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775776032.059251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776032.059255:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775776032.059256:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776032.059260:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776032.059262:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776032.059263:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776032.059266:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776032.059268:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775776032.059270:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12914 00000100:00000001:2.0:1775776032.059273:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776032.059281:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776032.059288:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776032.059292:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776032.059296:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776032.059299:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776032.059301:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776032.059303:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776032.059307:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776032.059309:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776032.059311:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776032.059314:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776032.059316:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776032.059319:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776032.059321:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776032.059323:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776032.059325:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776032.059327:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776032.059330:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776032.059334:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776032.059337:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776032.059340:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776032.059344:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776032.059347:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776032.059349:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776032.059351:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776032.059361:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73b80. 00000020:00000040:3.0:1775776032.059364:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776032.059369:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776032.059371:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776032.059374:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776032.059376:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776032.059381:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776032.059387:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000100:00000001:3.0:1775776032.059394:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776032.059402:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73b80 x1862022710383872/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776032.059411:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776032.059415:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776032.059419:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c52008f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73b80 x1862022710383872/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776032.059439:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776032.059442:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776032.059446:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776032.059447:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776032.059449:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776032.059451:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776032.059454:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12914 00000400:00000200:3.0:1775776032.059459:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776032.059462:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776032.059464:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776032.059469:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776032.059472:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776032.059475:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776032.059478:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776032.059481:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776032.059482:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776032.059485:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776032.059488:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776032.059492:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776032.059495:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776032.059501:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776032.059503:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776032.059506:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776032.059507:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776032.059510:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776032.059512:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776032.059516:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776032.059517:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776032.059522:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776032.059525:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776032.059527:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776032.059530:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776032.059534:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776032.059535:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776032.059539:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775776032.059543:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776032.059545:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776032.059546:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776032.059548:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776032.059549:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776032.059552:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776032.059555:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000001:3.0:1775776032.059561:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776032.059567:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710384000/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776032.059578:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776032.059579:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776032.059584:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710384000/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776032.059590:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776032.059603:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776032.059608:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776032.059610:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776032.059613:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776032.059615:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776032.059619:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12914 00000400:00000200:3.0:1775776032.059623:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776032.059626:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776032.059629:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776032.059633:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776032.059636:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776032.059639:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776032.059643:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776032.059645:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776032.059646:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776032.059649:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776032.059650:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776032.059654:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776032.059657:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776032.059660:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776032.059662:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776032.059665:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776032.059668:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776032.059669:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776032.059671:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776032.059673:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776032.059675:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776032.059678:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776032.059680:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776032.059681:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776032.059683:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776032.059685:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776032.059687:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776032.059690:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775776032.059693:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776032.059697:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776032.059698:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776032.059699:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776032.059701:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776032.059704:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776032.059707:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000001:3.0:1775776032.059712:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776032.059717:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710384128/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776032.059727:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776032.059730:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776032.059734:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710384128/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776032.059742:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776032.059746:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776032.059748:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776032.059750:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776032.059755:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776032.059760:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12924) 00000800:00000200:1.1:1775776032.059790:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776032.059793:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775776032.060189:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776032.060191:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775776032.060469:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776032.060472:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00010000:00000001:3.0:1775776032.187265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776032.187270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776032.187275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776032.187276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776032.187280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776032.187281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776032.187350:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776032.187354:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776032.187357:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776033.083232:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776033.083237:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776033.083238:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776033.083242:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776033.083245:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776033.083247:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776033.083249:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776033.083251:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776033.083252:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776033.211212:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776033.211219:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776033.211224:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776033.211225:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776033.211229:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776033.211230:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776033.211284:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776033.211288:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776033.211291:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776034.107217:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776034.107222:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776034.107223:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776034.107227:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776034.107230:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776034.107233:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776034.107235:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776034.107237:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776034.107239:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776034.235254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776034.235260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776034.235265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776034.235267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776034.235270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776034.235271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776034.235337:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776034.235341:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776034.235344:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776035.131235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776035.131240:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776035.131241:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776035.131245:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776035.131248:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776035.131251:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776035.131253:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776035.131255:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776035.131257:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776035.259242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776035.259249:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776035.259251:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776035.259254:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776035.259258:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776035.259261:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776035.259263:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776035.259265:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776035.259269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776035.259270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776035.259274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776035.259275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776035.963282:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307590918, at 4307590976 00000100:00000010:0.0:1775776035.963300:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2360. 00000100:00000001:2.0:1775776036.155288:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776036.155295:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776036.155296:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776036.155307:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776036.155311:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776036.155314:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776036.155316:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776036.155319:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776036.155322:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776036.283262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776036.283267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776036.283272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776036.283273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776036.283277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776036.283279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776037.179224:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776037.179230:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776037.179231:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776037.179235:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776037.179239:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776037.179242:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776037.179245:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776037.179249:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776037.179253:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776037.179460:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776037.179465:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776037.179469:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776037.179479:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491aa00 x1862022710384256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.179496:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776037.179497:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776037.179511:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776037.179514:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776037.179518:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776037.179522:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710384256:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776037.179526:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776037.179533:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776037.179535:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776037.179540:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776037.179546:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775776037.179548:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776037.179556:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776037.179560:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775776037.179566:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776037.179572:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710384256, portal 25 00000100:00000001:0.0:1775776037.179575:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776037.179577:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684369408 : -123823025182208 : ffff8f623491aa00) 00000100:00000040:0.0:1775776037.179583:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491aa00 x1862022710384256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.179605:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776037.179608:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776037.179611:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775776037.179615:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710384256, offset 0 00000400:00000200:0.0:1775776037.179623:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776037.179632:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776037.179638:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776037.179644:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776037.179649:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776037.179654:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776037.179658:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776037.179661:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453554:453554:256:4294967295] 192.168.201.115@tcp LPNI seq info [453479:453479:8:4294967295] 00000400:00000200:0.0:1775776037.179663:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776037.179672:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776037.179681:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776037.179688:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775776037.179699:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776037.179705:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775776037.179714:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776037.179720:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775776037.179729:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776037.179732:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775776037.179737:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776037.179745:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776037.179752:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775776037.179756:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775776037.179760:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776037.179765:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491aa00 x1862022710384256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.179778:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776037.179782:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491aa00 x1862022710384256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776062 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.179792:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776037.179794:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776037.179800:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776037.179806:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776037.179810:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307596038 00000100:00000010:0.0:1775776037.179815:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c36c0. 00000100:00000001:0.0:1775776037.179821:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776037.179828:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307596038 00000100:00000010:0.0:1775776037.179832:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3840. 00000100:00000001:0.0:1775776037.179836:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776037.179840:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776037.179844:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776037.179849:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776037.179856:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776037/real 1775776037] req@ffff8f623491aa00 x1862022710384256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776037.179873:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776037.179875:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776037.179878:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775776037.179880:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776037.179883:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776037.179885:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776037.179889:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491aa00 x1862022710384256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776037.179898:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491aa00 x1862022710384256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.179904:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776037.179905:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776037.179906:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776037.179907:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776037.179909:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491aa00 x1862022710384256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.179914:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776037.179916:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491aa00 x1862022710384256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.179920:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776037.179922:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776037.179923:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776037.179924:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776037.179925:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776037.179926:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776037.179928:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776037.179930:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776037.179931:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776037.179934:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491aa00 x1862022710384256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776037.179938:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710384256:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776037.179941:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776037.179942:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776037.179943:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491aa00 x1862022710384256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.179947:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776037.179948:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776037.179949:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775776037.179951:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776037.179952:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776037.179953:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776037.179954:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776037.179955:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:0.0:1775776037.179957:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776037.179958:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776037.179959:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491aa00. 00000100:00000001:0.0:1775776037.179961:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776037.179962:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776037.179963:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776037.179968:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776037.179969:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776037.179970:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776037.179971:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776037.179972:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776037.179975:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918a80 x1862022710384384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.179979:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776037.179980:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776037.179981:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776037.179982:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776037.179983:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776037.179985:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710384384:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776037.179987:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776037.179988:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776037.179989:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776037.179990:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776037.179991:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775776037.179993:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776037.179995:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776037.179996:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775776037.179998:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776037.180000:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710384384, portal 10 00000100:00000001:0.0:1775776037.180001:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776037.180002:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684361344 : -123823025190272 : ffff8f6234918a80) 00000100:00000040:0.0:1775776037.180006:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918a80 x1862022710384384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.180009:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776037.180011:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776037.180012:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775776037.180013:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710384384, offset 0 00000400:00000200:0.0:1775776037.180015:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776037.180017:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776037.180019:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776037.180021:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776037.180023:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776037.180024:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776037.180026:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776037.180028:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453555:453555:256:4294967295] 192.168.201.115@tcp LPNI seq info [453480:453480:8:4294967295] 00000400:00000200:0.0:1775776037.180030:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776037.180034:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776037.180037:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776037.180039:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775776037.180041:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776037.180043:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775776037.180045:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776037.180047:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775776037.180049:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776037.180050:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775776037.180052:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776037.180053:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776037.180055:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775776037.180056:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775776037.180057:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776037.180059:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918a80 x1862022710384384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.180064:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776037.180066:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918a80 x1862022710384384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776062 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.180069:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776037.180070:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776037.180072:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776037.180074:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776037.180076:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776037.180077:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307592203 00000100:00000001:0.0:1775776037.180079:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307596038 00000100:00000001:0.0:1775776037.180080:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307592203 00000100:00000001:0.0:1775776037.180081:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307596038 00000100:00000001:0.0:1775776037.180083:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776037.180084:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776037.180085:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776037.180087:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776037/real 1775776037] req@ffff8f6234918a80 x1862022710384384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776037.180091:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776037.180092:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776037.180092:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775776037.180093:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776037.180094:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776037.180095:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776037.180097:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918a80 x1862022710384384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776037.180101:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918a80 x1862022710384384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.180105:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776037.180106:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776037.180106:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776037.180107:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776037.180109:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918a80 x1862022710384384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.180112:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776037.180114:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918a80 x1862022710384384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.180117:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776037.180119:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776037.180119:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776037.180120:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776037.180121:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776037.180124:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776037.180126:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776037.180128:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776037.180130:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776037.180132:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918a80 x1862022710384384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776037.180161:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710384384:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776037.180164:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776037.180166:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491b800 x1862022710384512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.180170:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776037.180171:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776037.180173:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776037.180174:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776037.180175:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776037.180176:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710384512:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776037.180178:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776037.180179:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776037.180180:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776037.180181:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776037.180184:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776037.180185:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776037.180187:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776037.180188:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775776037.180190:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776037.180192:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710384512, portal 10 00000100:00000001:0.0:1775776037.180193:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776037.180194:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684372992 : -123823025178624 : ffff8f623491b800) 00000100:00000040:0.0:1775776037.180197:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491b800 x1862022710384512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.180205:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776037.180206:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776037.180208:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775776037.180209:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710384512, offset 0 00000400:00000200:0.0:1775776037.180211:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776037.180213:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776037.180215:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776037.180217:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776037.180218:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776037.180219:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776037.180221:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776037.180223:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453556:453556:256:4294967295] 192.168.201.115@tcp LPNI seq info [453481:453481:8:4294967295] 00000400:00000200:0.0:1775776037.180225:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776037.180229:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776037.180231:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776037.180233:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775776037.180235:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776037.180236:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775776037.180239:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776037.180240:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775776037.180242:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776037.180243:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775776037.180245:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776037.180247:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776037.180249:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775776037.180250:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775776037.180251:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776037.180253:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491b800 x1862022710384512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.180257:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776037.180260:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491b800 x1862022710384512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776062 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.180272:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776037.180273:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776037.180276:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776037.180278:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776037.180281:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776037.180283:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307592204 00000100:00000001:0.0:1775776037.180286:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307596038 00000100:00000001:0.0:1775776037.180288:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307592204 00000100:00000001:0.0:1775776037.180290:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307596038 00000100:00000001:0.0:1775776037.180292:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776037.180294:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776037.180296:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776037.180300:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776037/real 1775776037] req@ffff8f623491b800 x1862022710384512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776037.180308:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776037.180309:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776037.180311:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775776037.180313:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776037.180314:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776037.180316:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776037.180320:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491b800 x1862022710384512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776037.180325:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491b800 x1862022710384512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.180329:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776037.180330:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776037.180331:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776037.180332:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776037.180335:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491b800 x1862022710384512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.180342:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776037.180346:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491b800 x1862022710384512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.180352:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776037.180354:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776037.180355:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776037.180357:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776037.180358:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776037.180360:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776037.180363:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776037.180365:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776037.180367:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776037.180371:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491b800 x1862022710384512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776037.180378:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710384512:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776037.180381:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776037.180382:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776037.180385:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918a80 x1862022710384384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.180391:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776037.180392:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776037.180393:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6c00. 02000000:00000001:0.0:1775776037.180396:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776037.180397:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776037.180398:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776037.180400:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776037.180401:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4fc00. 02000000:00000001:0.0:1775776037.180403:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776037.180405:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776037.180406:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918a80. 00000100:00000001:0.0:1775776037.180408:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776037.180409:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776037.180410:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776037.180413:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491b800 x1862022710384512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776062 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776037.180418:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776037.180419:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776037.180420:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776037.180422:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776037.180423:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776037.180424:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776037.180426:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776037.180428:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4f400. 02000000:00000001:0.0:1775776037.180429:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776037.180431:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776037.180432:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491b800. 00000100:00000001:0.0:1775776037.180434:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776037.180435:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776037.180436:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776037.180439:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776037.180440:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776037.180441:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776037.180443:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776037.307199:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776037.307204:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776037.307208:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776037.307208:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776037.307211:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776037.307212:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776037.307276:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776037.307278:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776037.307281:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776038.203242:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776038.203249:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776038.203251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776038.203256:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776038.203260:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776038.203264:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776038.203266:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776038.203269:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776038.203272:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776038.331250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776038.331255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776038.331261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776038.331262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776038.331266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776038.331267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776038.331331:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776038.331334:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776038.331336:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776039.227248:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776039.227255:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776039.227257:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776039.227263:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776039.227266:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776039.227278:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776039.227280:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776039.227282:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776039.227284:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776039.355231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776039.355236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776039.355239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776039.355242:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776039.355250:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776039.355252:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776039.355254:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776039.355255:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776039.355257:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776039.355258:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776039.355261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776039.355262:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776039.355263:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776039.355264:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776039.355265:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776039.355266:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776039.355268:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776039.355270:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776039.355324:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776039.355327:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776039.355329:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776040.379251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776040.379257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776040.379262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776040.379263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776040.379266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776040.379267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776041.083231:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307596038, at 4307596096 00000100:00000010:0.0:1775776041.083249:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c36c0. 00010000:00000001:3.0:1775776041.403244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776041.403248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776041.403252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776041.403253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776041.403256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776041.403257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776042.299237:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776042.299245:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776042.299248:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776042.299251:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776042.299255:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776042.299262:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12924 00000400:00000200:0.0:1775776042.299271:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776042.299275:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776042.299278:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776042.299282:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776042.299285:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776042.299288:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776042.299290:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776042.299294:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776042.299295:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776042.299298:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776042.299300:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776042.299302:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776042.299305:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776042.299308:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776042.299310:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776042.299312:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776042.299313:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776042.299316:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776042.299319:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776042.299322:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776042.299325:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776042.299328:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776042.299330:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776042.299332:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776042.299335:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776042.299342:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060e00. 00000020:00000040:0.0:1775776042.299346:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776042.299349:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776042.299351:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776042.299353:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776042.299356:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776042.299359:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776042.299366:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7800. 00000100:00000001:0.0:1775776042.299372:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776042.299379:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060e00 x1862022710384640/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776042.299387:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776042.299390:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776042.299393:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004a3bddaa] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060e00 x1862022710384640/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776042.299409:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776042.299411:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776042.299435:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776042.299436:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776042.299437:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776042.299439:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776042.299442:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12924 00000400:00000200:0.0:1775776042.299447:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776042.299450:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776042.299454:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776042.299458:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776042.299460:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776042.299464:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776042.299468:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776042.299471:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776042.299473:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776042.299475:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776042.299477:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775776042.299480:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775776042.299482:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775776042.299486:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776042.299490:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776042.299492:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776042.299494:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776042.299497:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775776042.299499:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776042.299502:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776042.299504:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776042.299508:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776042.299512:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776042.299514:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776042.299517:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776042.299520:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776042.299521:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776042.299525:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063100. 00000020:00000040:0.0:1775776042.299529:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776042.299531:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776042.299534:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776042.299535:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776042.299536:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776042.299537:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776042.299541:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6c00. 00000100:00000001:0.0:1775776042.299544:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776042.299550:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063100 x1862022710384768/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776042.299559:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776042.299561:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776042.299565:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033f6db7f] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063100 x1862022710384768/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776042.299571:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776042.299574:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776042.299577:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776042.299578:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776042.299579:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776042.299581:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776042.299585:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12924 00000400:00000200:0.0:1775776042.299589:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776042.299593:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776042.299596:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776042.299600:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776042.299604:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776042.299608:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776042.299612:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776042.299617:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776042.299620:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776042.299623:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776042.299626:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775776042.299630:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775776042.299634:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775776042.299640:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776042.299643:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776042.299645:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776042.299655:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776042.299659:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775776042.299664:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776042.299668:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776042.299672:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776042.299675:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776042.299681:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776042.299683:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776042.299686:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776042.299689:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776042.299692:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776042.299699:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061c00. 00000020:00000040:0.0:1775776042.299703:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776042.299707:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776042.299710:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776042.299714:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776042.299716:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776042.299718:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776042.299724:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5800. 00000100:00000001:0.0:1775776042.299729:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776042.299737:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061c00 x1862022710384896/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776042.299747:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776042.299749:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776042.299757:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076a44879] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061c00 x1862022710384896/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776042.299771:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776042.299771:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:0.0:1775776042.299776:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775776042.299777:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000100:00000040:0.0:1775776042.299782:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776042.299788:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776042.299793:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776042.299800:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12935) 00000800:00000200:1.1:1775776042.300175:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776042.300177:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775776042.300461:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776042.300464:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00010000:00000001:3.0:1775776042.427218:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776042.427223:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776042.427227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776042.427228:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776042.427231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776042.427232:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776042.427287:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776042.427290:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776042.427293:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776043.323223:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776043.323229:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776043.323231:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776043.323235:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776043.323238:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776043.323241:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776043.323244:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776043.323245:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776043.323248:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776043.451241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776043.451246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776043.451252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776043.451253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776043.451257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776043.451258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776043.451320:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776043.451324:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776043.451328:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776044.347205:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776044.347212:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776044.347213:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776044.347218:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776044.347221:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776044.347225:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776044.347227:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776044.347229:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776044.347232:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776044.475217:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776044.475224:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776044.475226:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776044.475230:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776044.475234:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776044.475237:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776044.475238:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776044.475240:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776044.475244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776044.475245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776044.475248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776044.475250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776044.475337:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776044.475340:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776044.475343:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776045.371230:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776045.371234:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776045.371235:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776045.371239:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776045.371242:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776045.371245:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776045.371247:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776045.371248:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776045.371250:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776045.499242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776045.499247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776045.499250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776045.499251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776045.499254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776045.499255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776046.203268:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307601158, at 4307601216 00000100:00000010:0.0:1775776046.203282:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2420. 00000100:00000001:2.0:1775776046.395241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776046.395247:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776046.395249:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776046.395254:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776046.395257:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776046.395261:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776046.395264:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776046.395266:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776046.395269:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776046.523256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776046.523262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776046.523267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776046.523269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776046.523273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776046.523274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776047.419230:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776047.419236:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776047.419238:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776047.419243:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776047.419247:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776047.419250:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775776047.419251:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776047.419252:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776047.419254:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775776047.419256:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776047.419257:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776047.419259:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776047.419261:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776047.419264:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776047.419271:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12930 00000400:00000200:3.0:1775776047.419279:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776047.419285:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776047.419287:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776047.419290:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776047.419293:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776047.419295:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776047.419296:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776047.419299:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776047.419301:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776047.419303:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776047.419305:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776047.419306:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776047.419308:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776047.419309:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776047.419310:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776047.419311:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776047.419312:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776047.419314:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776047.419316:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776047.419318:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776047.419320:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776047.419331:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776047.419334:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776047.419336:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776047.419338:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776047.419345:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775776047.419348:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776047.419351:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776047.419352:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776047.419354:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776047.419356:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776047.419359:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776047.419365:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775776047.419369:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776047.419376:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710385024/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776047.419383:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776047.419386:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776047.419389:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710385024/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776047.419406:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776047.419408:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776047.419414:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776047.419415:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776047.419416:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776047.419418:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776047.419420:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12930 00000400:00000200:3.0:1775776047.419422:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776047.419424:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776047.419426:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776047.419428:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776047.419430:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776047.419432:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776047.419433:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776047.419435:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776047.419436:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776047.419437:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776047.419439:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776047.419440:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776047.419441:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776047.419445:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776047.419448:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776047.419450:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776047.419452:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776047.419455:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776047.419459:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776047.419463:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776047.419466:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776047.419472:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776047.419474:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776047.419476:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776047.419481:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776047.419485:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776047.419489:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776047.419491:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775776047.419494:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776047.419497:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776047.419499:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776047.419500:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776047.419502:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776047.419503:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776047.419507:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:3.0:1775776047.419512:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776047.419516:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710385152/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776047.419529:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776047.419530:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776047.419534:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710385152/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776047.419546:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776047.419550:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776047.419556:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776047.419559:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776047.419561:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776047.419566:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776047.419569:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12930 00000400:00000200:3.0:1775776047.419575:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776047.419580:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776047.419583:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776047.419584:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776047.419587:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776047.419590:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776047.419591:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776047.419593:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776047.419593:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776047.419596:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776047.419599:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776047.419601:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776047.419603:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776047.419608:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776047.419610:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776047.419614:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776047.419617:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776047.419619:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776047.419623:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776047.419627:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776047.419628:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776047.419631:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776047.419636:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776047.419639:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776047.419642:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776047.419644:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776047.419645:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776047.419651:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000020:00000040:3.0:1775776047.419655:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776047.419657:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776047.419658:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776047.419661:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776047.419662:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776047.419663:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776047.419668:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000001:3.0:1775776047.419672:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776047.419678:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00100000:3.0:1775776047.419679:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710385280/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776047.419685:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776047.419686:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776047.419689:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710385280/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776047.419693:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776047.419696:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776047.419701:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776047.419704:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776047.419707:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776047.419712:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12940) 00000800:00000200:1.1:1775776047.419852:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776047.419962:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00010000:00000001:3.0:1775776047.547251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776047.547255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776047.547258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776047.547259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776047.547262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776047.547263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776047.547314:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776047.547318:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776047.547338:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776048.443237:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776048.443242:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776048.443243:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776048.443247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776048.443250:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776048.443253:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776048.443255:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776048.443257:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776048.443259:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776048.571270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776048.571276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776048.571281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776048.571282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776048.571285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776048.571285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776048.571352:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776048.571355:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776048.571358:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776049.467234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776049.467240:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776049.467242:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776049.467247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776049.467250:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776049.467254:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776049.467258:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776049.467261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776049.467264:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776049.595312:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776049.595322:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776049.595330:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776049.595335:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776049.595337:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776049.595342:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776049.595346:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776049.595348:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776049.595350:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776049.595352:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776049.595355:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776049.595356:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776049.595357:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776049.595358:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776049.595359:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776049.595360:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776049.595361:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776049.595362:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776049.595408:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776049.595411:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776049.595414:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775776050.491445:0:2402:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:0.0:1775776050.491483:0:2402:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1775776050.491493:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776050.491495:0:2402:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:0.0:1775776050.491499:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776050.491504:0:2402:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:0.0:1775776050.491509:0:2402:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:0.0:1775776050.491514:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:0.0:1775776050.491520:0:2402:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776050.491525:0:2402:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776050.491533:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3f68 00000400:00000010:0.0:1775776050.491535:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3f68. 00000100:00000001:0.0:1775776050.491543:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776050.491554:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c71180 x1862022710385024/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776050.491579:0:2402:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776050.491583:0:2402:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71180 x1862022710385024/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776072 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776050.491591:0:2402:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776050.491593:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775776050.491604:0:2402:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:0.0:1775776050.491606:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230a00. 00000400:00000200:0.0:1775776050.491609:0:2402:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776050.491613:0:2402:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776050.491617:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3110 00000400:00000010:0.0:1775776050.491620:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3110. 00000100:00000001:0.0:1775776050.491623:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776050.491630:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c70700 x1862022710385152/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776050.491645:0:2402:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776050.491651:0:2402:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70700 x1862022710385152/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776072 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776050.491661:0:2402:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776050.491664:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775776050.491670:0:2402:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:0.0:1775776050.491674:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230700. 00000400:00000200:0.0:1775776050.491679:0:2402:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776050.491686:0:2402:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776050.491691:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3a18 00000400:00000010:0.0:1775776050.491694:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3a18. 00000100:00000001:0.0:1775776050.491699:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776050.491706:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c70a80 x1862022710385280/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776050.491724:0:2402:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776050.491729:0:2402:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70a80 x1862022710385280/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776072 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776050.491740:0:2402:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776050.491744:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:3.0:1775776050.619255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776050.619264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776050.619298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776050.619300:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776050.619304:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776050.619305:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776051.323247:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307606278, at 4307606336 00000100:00000010:0.0:1775776051.323277:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2600. 00010000:00000001:3.0:1775776051.643263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776051.643267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776051.643271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776051.643272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776051.643275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776051.643275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776052.539294:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776052.539302:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776052.539306:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776052.539310:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776052.539314:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776052.539325:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12935 00000400:00000200:0.0:1775776052.539334:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776052.539339:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776052.539342:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776052.539345:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776052.539347:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776052.539350:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776052.539352:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776052.539355:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776052.539357:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776052.539360:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776052.539362:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776052.539364:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776052.539366:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776052.539369:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776052.539371:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776052.539372:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776052.539374:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776052.539376:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776052.539378:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776052.539381:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776052.539384:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776052.539386:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776052.539389:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776052.539391:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776052.539392:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776052.539399:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061880. 00000020:00000040:0.0:1775776052.539402:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776052.539405:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776052.539407:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776052.539409:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776052.539412:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776052.539415:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776052.539422:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7c00. 00000100:00000001:0.0:1775776052.539427:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776052.539435:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061880 x1862022710385408/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776052.539444:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776052.539448:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776052.539453:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005afd1f27] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061880 x1862022710385408/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776052.539474:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776052.539477:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776052.539481:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776052.539483:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776052.539484:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776052.539487:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776052.539489:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12935 00000400:00000200:0.0:1775776052.539492:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776052.539495:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776052.539497:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776052.539500:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776052.539502:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776052.539503:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776052.539505:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776052.539507:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776052.539508:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776052.539510:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776052.539512:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775776052.539514:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775776052.539516:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775776052.539518:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776052.539520:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776052.539522:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776052.539524:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776052.539526:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775776052.539528:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776052.539531:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776052.539533:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776052.539538:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776052.539541:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776052.539542:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776052.539547:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776052.539552:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776052.539556:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776052.539562:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060700. 00000020:00000040:0.0:1775776052.539567:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776052.539573:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776052.539576:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776052.539580:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776052.539583:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776052.539587:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776052.539594:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4800. 00000100:00000001:0.0:1775776052.539599:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776052.539609:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060700 x1862022710385536/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776052.539622:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776052.539625:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776052.539633:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000225c8695] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060700 x1862022710385536/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776052.539649:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776052.539653:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776052.539658:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776052.539661:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776052.539664:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776052.539667:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776052.539671:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12935 00000400:00000200:0.0:1775776052.539678:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776052.539683:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776052.539687:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776052.539693:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776052.539698:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776052.539702:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776052.539706:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776052.539710:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776052.539713:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776052.539716:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776052.539719:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775776052.539724:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775776052.539727:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775776052.539732:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776052.539734:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776052.539736:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776052.539737:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776052.539740:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775776052.539745:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776052.539748:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776052.539751:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776052.539755:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776052.539759:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776052.539762:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776052.539764:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776052.539769:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776052.539772:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776052.539778:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061180. 00000020:00000040:0.0:1775776052.539782:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776052.539786:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776052.539788:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776052.539792:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776052.539795:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776052.539796:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776052.539802:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5c00. 00000100:00000001:0.0:1775776052.539807:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776052.539816:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061180 x1862022710385664/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776052.539826:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776052.539829:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776052.539836:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e23859] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061180 x1862022710385664/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1775776052.539849:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775776052.539851:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776052.539854:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000100:00000040:0.0:1775776052.539855:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776052.539862:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776052.539867:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776052.539873:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776052.539878:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12945) 00000800:00000200:1.1:1775776052.540282:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776052.540285:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775776052.540606:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776052.540609:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00010000:00000001:3.0:1775776052.667254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776052.667258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776052.667262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776052.667263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776052.667265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776052.667266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776052.667343:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776052.667349:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776052.667352:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776053.691290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776053.691297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776053.691302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776053.691303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776053.691306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776053.691307:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776053.691364:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776053.691368:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776053.691371:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776054.715244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776054.715253:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776054.715256:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776054.715260:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776054.715267:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776054.715269:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776054.715272:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776054.715274:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776054.715280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776054.715281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776054.715285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776054.715286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776054.715346:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776054.715350:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776054.715352:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776055.739286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776055.739292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776055.739297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776055.739298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776055.739302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776055.739303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776056.443279:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307611398, at 4307611456 00000100:00000010:0.0:1775776056.443293:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2720. 00010000:00000001:3.0:1775776056.763240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776056.763246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776056.763253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776056.763254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776056.763257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776056.763259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776057.659273:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776057.659281:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776057.659284:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776057.659288:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776057.659293:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776057.659301:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12940 00000400:00000200:2.0:1775776057.659311:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776057.659317:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776057.659320:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776057.659324:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776057.659327:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776057.659329:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776057.659331:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776057.659334:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776057.659336:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776057.659338:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776057.659340:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776057.659342:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776057.659344:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776057.659346:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776057.659347:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776057.659348:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776057.659349:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776057.659351:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776057.659353:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776057.659354:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776057.659356:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776057.659358:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776057.659360:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776057.659361:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776057.659363:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776057.659370:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775776057.659377:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776057.659380:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776057.659381:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776057.659383:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776057.659385:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776057.659389:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776057.659396:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775776057.659401:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776057.659409:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710385792/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776057.659420:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776057.659424:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776057.659429:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710385792/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776057.659447:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776057.659449:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776057.659451:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776057.659452:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776057.659454:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776057.659455:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776057.659457:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12940 00000400:00000200:2.0:1775776057.659460:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776057.659463:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776057.659465:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776057.659467:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776057.659469:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776057.659471:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776057.659472:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776057.659475:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776057.659476:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776057.659478:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776057.659479:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776057.659482:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776057.659484:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776057.659488:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776057.659491:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776057.659494:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776057.659497:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776057.659501:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776057.659506:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776057.659508:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776057.659511:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776057.659516:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776057.659519:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776057.659522:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776057.659526:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776057.659530:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776057.659533:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776057.659539:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775776057.659542:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776057.659546:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776057.659548:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776057.659551:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776057.659553:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776057.659555:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776057.659562:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775776057.659567:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776057.659575:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710385920/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776057.659586:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776057.659589:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776057.659597:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710385920/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776057.659606:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776057.659611:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776057.659617:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776057.659620:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776057.659623:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776057.659626:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776057.659631:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12940 00000400:00000200:2.0:1775776057.659637:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776057.659641:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776057.659643:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776057.659648:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776057.659650:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776057.659652:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776057.659654:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776057.659657:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776057.659658:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776057.659660:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776057.659661:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776057.659663:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776057.659664:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776057.659670:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776057.659672:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776057.659674:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776057.659675:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776057.659676:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776057.659680:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776057.659684:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776057.659685:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776057.659688:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776057.659690:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776057.659692:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776057.659694:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776057.659697:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776057.659698:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776057.659702:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775776057.659706:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776057.659709:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776057.659712:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776057.659714:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776057.659716:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776057.659718:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776057.659723:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775776057.659726:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776057.659731:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710386048/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776057.659743:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776057.659746:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776057.659750:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710386048/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776057.659761:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776057.659762:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776057.659766:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776057.659769:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776057.659773:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776057.659776:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12950) 00000800:00000200:1.1:1775776057.659786:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776057.659789:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775776057.660177:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776057.660178:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775776057.660439:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776057.660441:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00010000:00000001:3.0:1775776057.787243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776057.787248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776057.787251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776057.787252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776057.787255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776057.787255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776057.787320:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776057.787323:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776057.787327:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776058.811245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776058.811250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776058.811256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776058.811257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776058.811261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776058.811262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776058.811330:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776058.811334:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776058.811337:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776059.835223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776059.835229:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776059.835235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776059.835238:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776059.835250:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776059.835254:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776059.835257:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776059.835262:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776059.835264:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776059.835266:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776059.835270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776059.835272:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776059.835273:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776059.835274:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776059.835276:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776059.835277:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776059.835278:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776059.835280:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776059.835344:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776059.835347:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776059.835349:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776060.859229:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776060.859234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776060.859237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776060.859238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776060.859241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776060.859242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776061.563204:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307616518, at 4307616576 00000100:00000010:0.0:1775776061.563217:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2a80. 00010000:00000001:3.0:1775776061.883207:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776061.883213:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776061.883218:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776061.883219:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776061.883222:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776061.883223:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776062.779230:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776062.779240:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776062.779244:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776062.779250:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776062.779256:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776062.779266:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12945 00000400:00000200:2.0:1775776062.779276:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776062.779291:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776062.779298:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776062.779302:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776062.779305:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776062.779308:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776062.779310:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776062.779313:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776062.779315:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776062.779317:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776062.779320:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776062.779323:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776062.779325:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776062.779328:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776062.779329:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776062.779331:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776062.779333:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776062.779336:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776062.779339:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776062.779341:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776062.779345:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776062.779348:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776062.779357:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776062.779360:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776062.779362:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776062.779371:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775776062.779374:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776062.779378:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776062.779380:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776062.779382:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776062.779384:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776062.779388:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776062.779395:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775776062.779401:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776062.779409:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710386176/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776062.779418:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776062.779422:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776062.779427:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710386176/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776062.779447:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776062.779450:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776062.779453:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776062.779454:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776062.779456:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776062.779459:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776062.779462:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12945 00000400:00000200:2.0:1775776062.779466:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776062.779471:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776062.779476:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776062.779483:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776062.779488:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776062.779493:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776062.779498:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776062.779503:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776062.779505:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776062.779508:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776062.779511:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776062.779516:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776062.779521:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776062.779528:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776062.779530:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776062.779534:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776062.779537:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776062.779541:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776062.779545:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776062.779549:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776062.779551:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776062.779557:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776062.779562:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776062.779567:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776062.779572:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776062.779578:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776062.779582:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776062.779588:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775776062.779593:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776062.779598:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776062.779601:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776062.779603:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776062.779605:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776062.779609:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776062.779615:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775776062.779618:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776062.779628:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710386304/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776062.779652:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776062.779657:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776062.779665:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710386304/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776062.779682:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776062.779686:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00080000:2.0:1775776062.779687:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775776062.779690:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000100:00080000:2.0:1775776062.779693:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776062.779697:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776062.779700:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776062.779705:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776062.779710:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12945 00000400:00000200:2.0:1775776062.779717:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776062.779723:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776062.779726:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776062.779730:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776062.779734:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776062.779739:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776062.779744:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776062.779747:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776062.779749:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776062.779752:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776062.779755:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776062.779759:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776062.779764:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776062.779771:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776062.779776:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776062.779781:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776062.779786:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776062.779789:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776062.779794:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776062.779797:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776062.779801:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776062.779806:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776062.779809:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776062.779813:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776062.779817:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776062.779821:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776062.779825:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776062.779831:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775776062.779835:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776062.779839:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776062.779842:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776062.779846:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776062.779849:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776062.779852:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776062.779858:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775776062.779862:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776062.779872:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710386432/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776062.779886:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776062.779890:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776062.779898:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710386432/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776062.779928:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776062.779932:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776062.779940:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776062.779946:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776062.779953:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776062.779960:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12955) 00000800:00000200:1.1:1775776062.780114:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776062.780116:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775776062.780368:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776062.780370:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00010000:00000001:3.0:1775776062.907246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776062.907251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776062.907255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776062.907256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776062.907260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776062.907261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776062.907315:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776062.907318:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776062.907321:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776063.931230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776063.931234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776063.931237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776063.931238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776063.931249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776063.931251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776063.931305:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776063.931309:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776063.931312:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776064.955247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776064.955254:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776064.955257:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776064.955260:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776064.955264:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776064.955266:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776064.955268:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776064.955270:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776064.955273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776064.955275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776064.955279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776064.955280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776064.955329:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776064.955332:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776064.955335:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776065.979299:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776065.979307:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776065.979315:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776065.979317:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776065.979321:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776065.979322:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776066.683208:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307621638, at 4307621696 00000100:00000010:0.0:1775776066.683226:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2f60. 00000800:00000200:2.0:1775776066.683235:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775776066.683240:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775776066.683246:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1775776066.683409:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776066.683426:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775776066.683431:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776066.683433:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1775776067.003270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776067.003277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776067.003283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776067.003284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776067.003288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776067.003289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776067.899263:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776067.899271:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776067.899274:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776067.899279:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776067.899284:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776067.899291:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12950 00000400:00000200:3.0:1775776067.899309:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776067.899314:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776067.899317:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776067.899320:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776067.899323:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776067.899325:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776067.899326:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776067.899329:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776067.899330:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776067.899333:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776067.899335:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776067.899336:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776067.899338:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776067.899340:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776067.899341:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776067.899342:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776067.899343:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776067.899346:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776067.899348:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776067.899350:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776067.899353:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776067.899356:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776067.899358:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776067.899360:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776067.899361:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776067.899370:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775776067.899373:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776067.899376:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776067.899381:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776067.899384:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776067.899386:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776067.899390:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776067.899399:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:3.0:1775776067.899405:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776067.899412:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710386560/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776067.899420:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776067.899424:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776067.899427:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710386560/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776067.899441:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776067.899444:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776067.899447:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776067.899449:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776067.899450:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776067.899454:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776067.899456:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12950 00000400:00000200:3.0:1775776067.899461:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776067.899464:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776067.899466:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776067.899471:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776067.899476:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776067.899480:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776067.899484:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776067.899490:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776067.899492:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776067.899496:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776067.899500:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776067.899503:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776067.899505:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776067.899511:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776067.899515:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776067.899519:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776067.899521:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776067.899523:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776067.899525:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776067.899527:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776067.899530:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776067.899536:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776067.899538:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776067.899541:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776067.899547:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776067.899550:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776067.899552:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776067.899556:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775776067.899558:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776067.899560:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776067.899562:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776067.899564:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776067.899565:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776067.899567:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776067.899570:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000100:00000001:3.0:1775776067.899575:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776067.899583:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710386688/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776067.899595:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776067.899598:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776067.899603:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710386688/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776067.899614:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776067.899616:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776067.899619:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776067.899620:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776067.899623:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776067.899627:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776067.899629:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12950 00000400:00000200:3.0:1775776067.899635:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776067.899640:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776067.899642:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776067.899644:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776067.899646:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776067.899647:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776067.899648:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776067.899651:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776067.899653:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776067.899655:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776067.899656:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776067.899658:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776067.899661:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776067.899665:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776067.899668:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776067.899671:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776067.899673:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776067.899676:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776067.899680:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776067.899684:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776067.899694:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776067.899697:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776067.899699:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776067.899700:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776067.899703:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776067.899706:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776067.899709:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776067.899711:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775776067.899714:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776067.899717:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776067.899718:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776067.899720:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776067.899722:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776067.899723:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776067.899726:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000001:3.0:1775776067.899727:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776067.899730:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710386816/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776067.899739:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776067.899740:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776067.899744:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710386816/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776067.899758:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776067.899762:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776067.899768:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775776067.899769:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776067.899773:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775776067.899774:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000100:00000040:3.0:1775776067.899781:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776067.899787:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12960) 00000800:00000200:1.1:1775776067.900246:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776067.900248:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775776067.900533:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776067.900535:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00010000:00000001:3.0:1775776068.027242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776068.027248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776068.027253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776068.027255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776068.027259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776068.027260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776068.027329:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776068.027333:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776068.027337:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776069.051279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776069.051287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776069.051294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776069.051295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776069.051299:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776069.051300:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776069.051371:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776069.051375:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776069.051379:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776070.075247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776070.075252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776070.075256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776070.075259:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776070.075260:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776070.075263:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776070.075265:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776070.075266:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776070.075268:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776070.075269:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776070.075272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776070.075273:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776070.075273:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776070.075274:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776070.075275:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776070.075276:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776070.075277:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776070.075278:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776070.075346:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776070.075349:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776070.075351:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776071.099263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776071.099269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776071.099273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776071.099274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776071.099278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776071.099279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776071.803237:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307626758, at 4307626816 00000100:00000010:0.0:1775776071.803259:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2120. 00010000:00000001:3.0:1775776072.123214:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776072.123219:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776072.123222:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776072.123223:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776072.123225:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776072.123226:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776073.019234:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776073.019241:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776073.019244:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776073.019247:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776073.019252:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776073.019259:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12955 00000400:00000200:2.0:1775776073.019268:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776073.019273:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776073.019275:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776073.019279:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776073.019282:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776073.019285:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776073.019287:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776073.019290:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776073.019292:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776073.019294:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776073.019297:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776073.019300:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776073.019302:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776073.019305:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776073.019307:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776073.019309:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776073.019311:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776073.019314:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776073.019317:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776073.019320:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776073.019322:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776073.019325:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776073.019328:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776073.019330:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776073.019332:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776073.019339:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775776073.019343:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776073.019347:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776073.019349:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776073.019351:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776073.019354:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776073.019357:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776073.019364:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775776073.019370:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776073.019378:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710386944/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776073.019388:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776073.019392:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776073.019396:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710386944/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776073.019417:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776073.019420:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776073.019423:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776073.019424:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776073.019425:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776073.019427:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776073.019429:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12955 00000400:00000200:2.0:1775776073.019432:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776073.019434:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776073.019435:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776073.019438:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776073.019439:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776073.019441:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776073.019442:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776073.019444:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776073.019445:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776073.019446:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776073.019447:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776073.019449:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776073.019450:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776073.019453:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776073.019456:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776073.019459:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776073.019471:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776073.019477:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776073.019485:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776073.019492:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776073.019495:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776073.019501:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776073.019505:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776073.019507:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776073.019511:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776073.019518:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776073.019522:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776073.019526:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775776073.019531:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776073.019535:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776073.019537:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776073.019539:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776073.019542:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776073.019545:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776073.019552:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775776073.019557:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776073.019563:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710387072/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776073.019570:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776073.019572:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776073.019577:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710387072/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776073.019593:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776073.019598:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776073.019607:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776073.019609:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776073.019612:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776073.019619:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776073.019623:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12955 00000400:00000200:2.0:1775776073.019630:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776073.019637:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776073.019640:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776073.019643:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776073.019648:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776073.019652:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776073.019656:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776073.019660:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776073.019663:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776073.019665:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776073.019668:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776073.019670:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776073.019673:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776073.019678:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776073.019681:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776073.019684:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776073.019687:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776073.019691:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776073.019695:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776073.019698:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776073.019701:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776073.019703:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776073.019710:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776073.019713:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776073.019715:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776073.019719:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776073.019720:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776073.019724:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775776073.019728:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776073.019730:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776073.019730:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776073.019732:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776073.019733:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776073.019735:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776073.019740:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775776073.019745:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776073.019753:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710387200/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1775776073.019757:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776073.019764:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000100:00000001:2.0:1775776073.019780:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776073.019784:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776073.019792:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710387200/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776073.019803:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776073.019805:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776073.019808:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776073.019811:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776073.019817:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776073.019825:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12965) 00000800:00000200:1.1:1775776073.020100:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776073.020102:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775776073.020331:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776073.020333:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00010000:00000001:3.0:1775776073.147253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776073.147259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776073.147264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776073.147265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776073.147269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776073.147270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776073.147338:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776073.147342:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776073.147345:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776074.171218:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776074.171223:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776074.171226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776074.171227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776074.171230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776074.171231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776074.171283:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776074.171287:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776074.171290:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776075.195249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776075.195256:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776075.195258:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776075.195261:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776075.195266:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776075.195268:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776075.195270:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776075.195272:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776075.195275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776075.195276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776075.195280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776075.195281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776075.195355:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776075.195359:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776075.195362:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776076.219238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776076.219244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776076.219249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776076.219250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776076.219254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776076.219255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776076.923220:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307631878, at 4307631936 00000100:00000010:0.0:1775776076.923236:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e25a0. 00010000:00000001:3.0:1775776077.243223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776077.243229:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776077.243234:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776077.243235:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776077.243239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776077.243240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776078.139211:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776078.139220:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776078.139225:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776078.139230:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776078.139235:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776078.139243:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12960 00000400:00000200:2.0:1775776078.139252:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776078.139258:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776078.139261:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776078.139265:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776078.139268:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776078.139271:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776078.139273:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776078.139277:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776078.139278:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776078.139281:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776078.139284:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776078.139286:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776078.139288:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776078.139291:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776078.139293:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776078.139295:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776078.139296:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776078.139298:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776078.139302:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776078.139304:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776078.139306:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776078.139310:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776078.139312:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776078.139315:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776078.139317:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776078.139339:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775776078.139343:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776078.139347:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776078.139349:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776078.139351:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776078.139354:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776078.139359:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776078.139376:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775776078.139383:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776078.139394:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710387328/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776078.139406:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776078.139411:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776078.139416:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710387328/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776078.139443:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776078.139448:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776078.139451:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776078.139452:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776078.139454:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776078.139457:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776078.139462:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12960 00000400:00000200:2.0:1775776078.139472:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776078.139483:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776078.139492:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776078.139501:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776078.139509:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776078.139519:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776078.139525:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776078.139532:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776078.139538:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776078.139542:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776078.139545:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776078.139549:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776078.139557:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776078.139568:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776078.139572:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776078.139581:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776078.139588:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776078.139595:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776078.139606:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776078.139615:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776078.139620:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776078.139631:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776078.139640:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776078.139647:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776078.139655:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000200:1.1:1775776078.139663:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776078.139666:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000008:00000001:2.0:1775776078.139677:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776078.139683:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776078.139693:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775776078.139700:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776078.139709:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776078.139713:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776078.139719:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776078.139724:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776078.139729:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776078.139739:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775776078.139749:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776078.139764:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710387456/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776078.139778:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776078.139784:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776078.139805:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710387456/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776078.139833:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776078.139840:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776078.139847:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776078.139852:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776078.139856:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776078.139863:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776078.139873:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12960 00000400:00000200:2.0:1775776078.139884:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776078.139893:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776078.139899:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776078.139907:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776078.139913:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776078.139921:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776078.139928:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776078.139932:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776078.139935:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776078.139940:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776078.139943:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776078.139949:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776078.139955:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776078.139965:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776078.139970:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776078.139976:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776078.139981:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776078.139985:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776078.139991:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776078.139996:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776078.140000:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776078.140004:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776078.140008:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776078.140011:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776078.140014:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776078.140018:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776078.140021:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776078.140027:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775776078.140031:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776078.140036:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776078.140039:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776078.140041:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776078.140044:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776078.140046:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776078.140051:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775776078.140057:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776078.140065:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710387584/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776078.140078:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776078.140081:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776078.140089:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710387584/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776078.140100:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776078.140105:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776078.140110:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776078.140117:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776078.140123:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776078.140131:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12970) 00000800:00000200:1.1:1775776078.140219:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776078.140220:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775776078.140410:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776078.140412:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00010000:00000001:3.0:1775776078.267226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776078.267231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776078.267234:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776078.267235:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776078.267238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776078.267238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776078.267303:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776078.267308:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776078.267311:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776079.291267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776079.291273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776079.291279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776079.291282:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776079.291284:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776079.291287:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776079.291289:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776079.291292:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776079.291293:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776079.291296:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776079.291300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776079.291301:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776079.291302:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776079.291303:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776079.291305:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776079.291306:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776079.291307:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776079.291309:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776079.291381:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776079.291385:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776079.291388:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776080.315268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776080.315273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776080.315278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776080.315279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776080.315283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776080.315284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776080.315356:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776080.315360:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776080.315363:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776081.339298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776081.339307:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776081.339316:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776081.339317:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776081.339322:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776081.339323:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776082.043282:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307636998, at 4307637056 00000100:00000010:0.0:1775776082.043305:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2300. 00000100:00000001:2.0:1775776082.235246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776082.235250:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776082.235251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776082.235255:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776082.235259:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776082.235262:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776082.235264:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776082.235266:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776082.235267:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776082.363288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776082.363303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776082.363311:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776082.363312:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776082.363316:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776082.363317:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776083.259253:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776083.259259:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776083.259264:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776083.259276:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776083.259287:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776083.259295:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776083.259302:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776083.259305:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776083.259309:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776083.259525:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776083.259529:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776083.259531:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776083.259540:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918000 x1862022710387712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.259560:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776083.259564:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776083.259568:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776083.259570:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776083.259576:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776083.259584:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710387712:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776083.259592:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776083.259598:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776083.259604:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776083.259608:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776083.259616:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776083.259622:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776083.259633:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776083.259640:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775776083.259651:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776083.259663:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710387712, portal 25 00000100:00000001:0.0:1775776083.259667:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776083.259669:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684358656 : -123823025192960 : ffff8f6234918000) 00000100:00000040:0.0:1775776083.259679:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918000 x1862022710387712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.259697:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776083.259713:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776083.259716:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775776083.259721:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710387712, offset 0 00000400:00000200:0.0:1775776083.259732:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776083.259745:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776083.259754:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776083.259764:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776083.259772:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776083.259780:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776083.259786:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776083.259794:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453581:453581:256:4294967295] 192.168.201.115@tcp LPNI seq info [453506:453506:8:4294967295] 00000400:00000200:0.0:1775776083.259805:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776083.259827:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776083.259838:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776083.259848:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775776083.259858:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776083.259863:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775776083.259869:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776083.259873:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1775776083.259877:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776083.259880:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775776083.259887:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776083.259892:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776083.259898:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775776083.259901:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775776083.259906:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776083.259910:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918000 x1862022710387712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.259919:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776083.259923:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918000 x1862022710387712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776108 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.259931:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776083.259933:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776083.259940:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776083.259944:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776083.259947:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307642118 00000100:00000010:0.0:1775776083.259952:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776083.259960:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776083.259967:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307642118 00000100:00000010:0.0:1775776083.259970:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c00. 00000100:00000001:0.0:1775776083.259973:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776083.259977:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776083.259980:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776083.259983:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776083.259987:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776083/real 1775776083] req@ffff8f6234918000 x1862022710387712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776083.260005:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776083.260006:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776083.260008:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775776083.260011:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776083.260012:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776083.260015:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776083.260018:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918000 x1862022710387712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776083.260027:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918000 x1862022710387712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.260032:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776083.260034:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776083.260036:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776083.260039:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776083.260042:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918000 x1862022710387712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.260049:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776083.260053:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918000 x1862022710387712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.260060:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776083.260063:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776083.260064:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776083.260065:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776083.260068:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776083.260070:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776083.260072:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776083.260076:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776083.260079:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776083.260084:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918000 x1862022710387712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776083.260092:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710387712:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776083.260097:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776083.260098:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776083.260101:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918000 x1862022710387712/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.260108:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776083.260111:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776083.260114:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776083.260117:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776083.260118:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776083.260120:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776083.260123:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776083.260125:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:0.0:1775776083.260129:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776083.260131:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776083.260168:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918000. 00000100:00000001:0.0:1775776083.260172:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776083.260174:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776083.260176:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776083.260185:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776083.260186:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776083.260188:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776083.260190:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776083.260191:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776083.260195:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491b800 x1862022710387840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.260202:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776083.260204:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776083.260206:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776083.260208:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776083.260211:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776083.260214:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710387840:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776083.260217:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776083.260219:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776083.260220:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776083.260222:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776083.260225:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776083.260227:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776083.260230:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776083.260233:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775776083.260237:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776083.260240:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710387840, portal 10 00000100:00000001:0.0:1775776083.260242:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776083.260243:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684372992 : -123823025178624 : ffff8f623491b800) 00000100:00000040:0.0:1775776083.260250:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491b800 x1862022710387840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.260256:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776083.260259:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776083.260261:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775776083.260263:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710387840, offset 0 00000400:00000200:0.0:1775776083.260265:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776083.260268:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776083.260271:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776083.260273:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776083.260274:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776083.260276:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776083.260278:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776083.260279:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453582:453582:256:4294967295] 192.168.201.115@tcp LPNI seq info [453507:453507:8:4294967295] 00000400:00000200:0.0:1775776083.260282:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776083.260286:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776083.260288:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776083.260291:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775776083.260293:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776083.260295:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775776083.260298:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776083.260300:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1775776083.260302:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776083.260303:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775776083.260305:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776083.260307:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776083.260309:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775776083.260311:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775776083.260312:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776083.260314:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491b800 x1862022710387840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.260320:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776083.260321:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491b800 x1862022710387840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776108 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.260326:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776083.260327:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776083.260328:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776083.260331:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776083.260333:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776083.260334:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307638284 00000100:00000001:0.0:1775776083.260336:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307642118 00000100:00000001:0.0:1775776083.260338:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307638284 00000100:00000001:0.0:1775776083.260339:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307642118 00000100:00000001:0.0:1775776083.260347:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776083.260349:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776083.260350:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776083.260352:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776083/real 1775776083] req@ffff8f623491b800 x1862022710387840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776083.260361:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776083.260362:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776083.260363:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775776083.260365:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776083.260365:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776083.260367:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776083.260368:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491b800 x1862022710387840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776083.260373:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491b800 x1862022710387840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.260377:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776083.260378:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776083.260379:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776083.260379:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776083.260381:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491b800 x1862022710387840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.260384:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776083.260386:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491b800 x1862022710387840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.260390:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776083.260391:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776083.260392:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776083.260393:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776083.260394:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776083.260397:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776083.260399:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776083.260401:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776083.260403:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776083.260405:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491b800 x1862022710387840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776083.260410:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710387840:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776083.260412:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776083.260414:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918a80 x1862022710387968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.260417:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776083.260418:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776083.260419:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776083.260420:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776083.260421:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776083.260423:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710387968:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776083.260424:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776083.260425:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776083.260426:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776083.260427:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776083.260429:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7000. 02000000:00000001:0.0:1775776083.260430:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776083.260432:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776083.260433:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775776083.260435:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776083.260437:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710387968, portal 10 00000100:00000001:0.0:1775776083.260438:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776083.260439:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684361344 : -123823025190272 : ffff8f6234918a80) 00000100:00000040:0.0:1775776083.260442:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918a80 x1862022710387968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.260447:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776083.260448:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776083.260449:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775776083.260450:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710387968, offset 0 00000400:00000200:0.0:1775776083.260453:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776083.260456:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776083.260459:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776083.260462:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776083.260464:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776083.260466:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776083.260468:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776083.260471:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453583:453583:256:4294967295] 192.168.201.115@tcp LPNI seq info [453508:453508:8:4294967295] 00000400:00000200:0.0:1775776083.260475:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776083.260481:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776083.260485:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776083.260488:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775776083.260492:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776083.260494:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775776083.260498:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776083.260501:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1775776083.260504:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776083.260506:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775776083.260509:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776083.260512:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776083.260515:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775776083.260517:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775776083.260519:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776083.260522:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918a80 x1862022710387968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.260527:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776083.260529:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918a80 x1862022710387968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776108 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.260532:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776083.260533:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776083.260535:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776083.260538:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776083.260540:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776083.260541:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307638284 00000100:00000001:0.0:1775776083.260543:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307642118 00000100:00000001:0.0:1775776083.260545:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307638284 00000100:00000001:0.0:1775776083.260546:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307642118 00000100:00000001:0.0:1775776083.260547:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776083.260548:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776083.260549:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776083.260551:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776083/real 1775776083] req@ffff8f6234918a80 x1862022710387968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776083.260555:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776083.260556:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776083.260557:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775776083.260559:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776083.260559:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776083.260560:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776083.260562:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918a80 x1862022710387968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776083.260567:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918a80 x1862022710387968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.260571:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776083.260571:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776083.260572:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776083.260573:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776083.260575:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918a80 x1862022710387968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.260578:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776083.260580:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918a80 x1862022710387968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.260583:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776083.260585:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776083.260586:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776083.260587:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776083.260588:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776083.260589:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776083.260591:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776083.260593:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776083.260594:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776083.260596:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918a80 x1862022710387968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776083.260600:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710387968:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776083.260602:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776083.260603:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776083.260605:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491b800 x1862022710387840/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.260609:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776083.260609:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776083.260611:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776083.260612:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776083.260613:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776083.260613:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776083.260615:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776083.260616:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4f400. 02000000:00000001:0.0:1775776083.260617:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776083.260618:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776083.260619:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491b800. 00000100:00000001:0.0:1775776083.260621:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776083.260622:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776083.260623:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776083.260624:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918a80 x1862022710387968/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776108 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776083.260632:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776083.260632:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776083.260633:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7000. 02000000:00000001:0.0:1775776083.260635:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776083.260635:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776083.260636:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776083.260637:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776083.260637:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4fc00. 02000000:00000001:0.0:1775776083.260638:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776083.260639:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776083.260639:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918a80. 00000100:00000001:0.0:1775776083.260640:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776083.260641:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776083.260642:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776083.260644:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776083.260645:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776083.260646:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776083.260647:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776083.387212:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776083.387219:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776083.387227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776083.387228:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776083.387233:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776083.387234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776083.387299:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776083.387304:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776083.387307:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776084.283252:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776084.283259:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776084.283260:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776084.283266:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776084.283269:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776084.283273:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776084.283276:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776084.283278:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776084.283280:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776084.411268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776084.411278:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776084.411280:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776084.411285:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776084.411291:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776084.411294:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776084.411296:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776084.411298:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776084.411303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776084.411305:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776084.411309:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776084.411311:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776084.411372:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776084.411377:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776084.411380:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776085.307246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776085.307253:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776085.307255:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776085.307260:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776085.307264:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776085.307268:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776085.307270:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776085.307272:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776085.307274:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776085.435272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776085.435280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776085.435286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776085.435288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776085.435293:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776085.435294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776085.435353:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776085.435358:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776085.435362:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776086.331269:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776086.331275:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776086.331277:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776086.331282:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776086.331285:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776086.331290:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776086.331292:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776086.331294:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776086.331296:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776086.459272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776086.459279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776086.459285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776086.459287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776086.459291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776086.459292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776087.163249:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307642118, at 4307642176 00000100:00000010:0.0:1775776087.163274:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3480. 00000100:00000001:2.0:1775776087.355263:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776087.355269:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776087.355270:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776087.355275:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776087.355279:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776087.355282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776087.355284:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776087.355286:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776087.355288:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776087.483270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776087.483277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776087.483285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776087.483286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776087.483290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776087.483291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776088.379270:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776088.379280:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776088.379283:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776088.379283:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775776088.379294:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:3.0:1775776088.379296:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776088.379303:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776088.379304:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775776088.379307:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775776088.379309:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776088.379311:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776088.379315:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776088.379318:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776088.379319:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776088.379333:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12970 00000400:00000200:3.0:1775776088.379343:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776088.379350:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776088.379353:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776088.379356:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776088.379359:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776088.379362:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776088.379364:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776088.379368:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776088.379370:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776088.379372:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776088.379375:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776088.379377:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776088.379380:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776088.379382:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776088.379384:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776088.379386:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776088.379388:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776088.379391:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776088.379394:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776088.379397:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776088.379400:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776088.379403:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776088.379406:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776088.379408:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776088.379410:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776088.379419:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775776088.379428:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776088.379432:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776088.379434:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776088.379437:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776088.379439:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776088.379443:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776088.379451:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000100:00000001:3.0:1775776088.379457:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776088.379465:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710388096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776088.379475:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776088.379480:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776088.379484:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710388096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776088.379504:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776088.379507:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776088.379511:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776088.379512:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776088.379514:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776088.379518:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776088.379521:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12970 00000400:00000200:3.0:1775776088.379527:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776088.379531:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:0.0:1775776088.379533:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1775776088.379534:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775776088.379539:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775776088.379543:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776088.379544:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776088.379549:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776088.379552:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776088.379557:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:0.0:1775776088.379559:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70380 x1862022710388096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775776088.379561:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776088.379564:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776088.379568:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776088.379571:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776088.379578:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776088.379584:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775776088.379591:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1775776088.379595:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776088.379597:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776088.379598:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776088.379604:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:0.0:1775776088.379605:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1775776088.379607:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:0.0:1775776088.379624:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776088.379629:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776088.379635:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710388096:192.168.201.115@tcp:250:kworker.0 00000020:00000040:3.0:1775776088.379636:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000001:0.0:1775776088.379645:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1775776088.379646:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776088.379651:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776088.379652:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776088.379654:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776088.379657:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776088.379661:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1775776088.379661:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776088.379667:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776088.379669:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000010:0.0:1775776088.379671:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7000. 00000008:00100000:3.0:1775776088.379673:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1775776088.379675:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775776088.379681:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776088.379683:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776088.379687:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000010:3.0:1775776088.379687:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000020:00000040:3.0:1775776088.379694:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000400:00000010:0.0:1775776088.379697:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 02000000:00000001:3.0:1775776088.379697:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776088.379699:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776088.379706:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776088.379707:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776088.379709:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776088.379711:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 02000000:00000010:3.0:1775776088.379713:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000100:00000001:3.0:1775776088.379719:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776088.379725:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710388224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1775776088.379728:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710388096, portal 25 00000100:00000001:0.0:1775776088.379734:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776088.379737:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004928 : -123823860546688 : ffff8f6202c70380) 00000100:00000001:3.0:1775776088.379746:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:0.0:1775776088.379751:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70380 x1862022710388096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776088.379752:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776088.379765:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710388224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.379767:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776088.379776:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776088.379783:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775776088.379789:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710388096, offset 0 00000100:00000001:3.0:1775776088.379789:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776088.379795:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775776088.379800:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00080000:3.0:1775776088.379801:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776088.379805:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776088.379806:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776088.379809:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:0.0:1775776088.379814:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00080000:3.0:1775776088.379816:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12970 00000400:00000200:3.0:1775776088.379823:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776088.379827:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:3.0:1775776088.379832:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776088.379839:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1775776088.379841:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1775776088.379842:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776088.379846:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776088.379848:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:0.0:1775776088.379852:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:3.0:1775776088.379853:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776088.379859:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776088.379860:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776088.379862:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:0.0:1775776088.379866:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000020:00000040:3.0:1775776088.379866:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776088.379876:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:0.0:1775776088.379880:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000020:00000001:3.0:1775776088.379881:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776088.379887:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776088.379892:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775776088.379896:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453584:453584:256:4294967295] 192.168.201.115@tcp LPNI seq info [453509:453509:8:4294967295] 00000100:00000040:3.0:1775776088.379898:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776088.379905:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776088.379911:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776088.379915:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000400:00000200:0.0:1775776088.379916:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776088.379922:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776088.379924:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776088.379928:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776088.379930:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776088.379931:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776088.379935:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776088.379940:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776088.379945:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:3.0:1775776088.379946:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776088.379954:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775776088.379957:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776088.379960:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776088.379963:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775776088.379967:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776088.379968:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776088.379972:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776088.379975:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775776088.379976:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 02000000:00000010:3.0:1775776088.379980:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000100:00000001:3.0:1775776088.379985:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775776088.379988:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776088.379994:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000100:00100000:3.0:1775776088.379994:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710388352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.1:1775776088.380007:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775776088.380010:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:0.1:1775776088.380013:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000100:00000001:3.0:1775776088.380015:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1775776088.380025:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776088.380026:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710388352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:0.0:1775776088.380032:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000100:00000001:3.0:1775776088.380041:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776088.380044:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:3.0:1775776088.380044:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776088.380053:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:0.0:1775776088.380058:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000040:3.0:1775776088.380060:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775776088.380065:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000100:00000040:3.0:1775776088.380065:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000010:0.0:1775776088.380070:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000040:3.0:1775776088.380074:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12981) 00000100:00000001:0.0:1775776088.380076:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776088.380083:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70380 x1862022710388096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380094:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776088.380098:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70380 x1862022710388096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776113 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380105:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776088.380107:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776088.380113:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776088.380117:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776088.380120:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307647238 00000100:00000010:0.0:1775776088.380125:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776088.380132:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776088.380166:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307647239 00000100:00000010:0.0:1775776088.380170:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3b40. 00000100:00000001:0.0:1775776088.380173:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776088.380177:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776088.380179:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776088.380182:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776088.380187:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776088/real 1775776088] req@ffff8f6202c70380 x1862022710388096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776088.380194:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776088.380196:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776088.380197:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775776088.380199:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776088.380200:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776088.380202:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776088.380205:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70380 x1862022710388096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776088.380211:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70380 x1862022710388096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380215:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776088.380216:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776088.380217:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776088.380218:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776088.380221:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70380 x1862022710388096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380224:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776088.380228:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70380 x1862022710388096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380232:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776088.380234:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776088.380235:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776088.380236:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776088.380238:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776088.380239:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776088.380241:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776088.380244:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776088.380245:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776088.380247:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70380 x1862022710388096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776088.380252:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710388096:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776088.380254:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776088.380256:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776088.380257:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70380 x1862022710388096/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380266:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776088.380268:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776088.380270:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7000. 02000000:00000001:0.0:1775776088.380273:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776088.380274:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776088.380275:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776088.380277:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776088.380279:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36c00. 02000000:00000001:0.0:1775776088.380282:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776088.380284:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776088.380285:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70380. 00000100:00000001:0.0:1775776088.380287:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776088.380288:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776088.380289:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776088.380296:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776088.380297:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776088.380298:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776088.380299:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776088.380300:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776088.380303:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70e00 x1862022710388224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380307:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776088.380308:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776088.380310:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776088.380312:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776088.380313:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776088.380314:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710388224:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776088.380316:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776088.380317:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776088.380318:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776088.380319:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776088.380321:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7000. 02000000:00000001:0.0:1775776088.380322:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776088.380324:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776088.380327:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775776088.380329:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776088.380331:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710388224, portal 10 00000100:00000001:0.0:1775776088.380332:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776088.380333:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849007616 : -123823860544000 : ffff8f6202c70e00) 00000100:00000040:0.0:1775776088.380337:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70e00 x1862022710388224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380342:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776088.380344:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776088.380345:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775776088.380348:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710388224, offset 0 00000400:00000200:0.0:1775776088.380351:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776088.380355:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776088.380358:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776088.380361:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776088.380363:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776088.380366:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776088.380368:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776088.380371:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453585:453585:256:4294967295] 192.168.201.115@tcp LPNI seq info [453510:453510:8:4294967295] 00000400:00000200:0.0:1775776088.380375:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776088.380382:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776088.380386:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776088.380389:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775776088.380393:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776088.380395:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775776088.380399:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776088.380401:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775776088.380404:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776088.380405:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775776088.380407:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776088.380409:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776088.380411:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775776088.380417:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775776088.380420:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776088.380422:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70e00 x1862022710388224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380427:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776088.380429:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70e00 x1862022710388224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776113 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380433:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776088.380434:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776088.380436:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776088.380439:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776088.380442:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776088.380443:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307643404 00000100:00000001:0.0:1775776088.380445:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307647238 00000100:00000001:0.0:1775776088.380447:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307643404 00000100:00000001:0.0:1775776088.380448:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307647239 00000100:00000001:0.0:1775776088.380449:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776088.380450:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776088.380451:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776088.380453:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776088/real 1775776088] req@ffff8f6202c70e00 x1862022710388224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776088.380457:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776088.380458:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776088.380460:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775776088.380461:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776088.380462:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776088.380463:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776088.380465:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70e00 x1862022710388224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776088.380470:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70e00 x1862022710388224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380478:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776088.380478:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776088.380480:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776088.380480:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776088.380482:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70e00 x1862022710388224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380486:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776088.380488:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70e00 x1862022710388224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380491:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776088.380492:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776088.380493:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776088.380494:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776088.380495:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776088.380499:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776088.380500:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776088.380502:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776088.380503:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776088.380505:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70e00 x1862022710388224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776088.380510:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710388224:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776088.380512:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776088.380514:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70000 x1862022710388352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380518:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776088.380518:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776088.380520:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776088.380521:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776088.380523:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776088.380524:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710388352:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776088.380527:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776088.380528:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776088.380529:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776088.380530:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776088.380532:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776088.380533:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776088.380534:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776088.380536:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775776088.380538:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776088.380539:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710388352, portal 10 00000100:00000001:0.0:1775776088.380541:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776088.380542:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004032 : -123823860547584 : ffff8f6202c70000) 00000100:00000040:0.0:1775776088.380545:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70000 x1862022710388352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380549:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776088.380550:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776088.380551:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775776088.380553:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710388352, offset 0 00000400:00000200:0.0:1775776088.380555:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776088.380557:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776088.380559:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776088.380560:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776088.380562:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776088.380563:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776088.380565:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776088.380567:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453586:453586:256:4294967295] 192.168.201.115@tcp LPNI seq info [453511:453511:8:4294967295] 00000400:00000200:0.0:1775776088.380569:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776088.380573:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776088.380576:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776088.380578:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775776088.380580:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776088.380582:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775776088.380585:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776088.380586:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775776088.380588:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776088.380590:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775776088.380591:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776088.380593:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776088.380595:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775776088.380596:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775776088.380598:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776088.380601:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70000 x1862022710388352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380607:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776088.380623:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70000 x1862022710388352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776113 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380629:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776088.380631:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776088.380634:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776088.380637:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776088.380639:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776088.380641:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307643404 00000100:00000001:0.0:1775776088.380642:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307647238 00000100:00000001:0.0:1775776088.380644:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307643404 00000100:00000001:0.0:1775776088.380645:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307647239 00000100:00000001:0.0:1775776088.380647:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776088.380648:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776088.380649:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776088.380651:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776088/real 1775776088] req@ffff8f6202c70000 x1862022710388352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776088.380660:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776088.380661:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776088.380663:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775776088.380665:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776088.380666:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776088.380668:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776088.380671:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70000 x1862022710388352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776088.380676:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70000 x1862022710388352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380680:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776088.380681:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776088.380682:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776088.380683:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776088.380684:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70000 x1862022710388352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380689:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776088.380691:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70000 x1862022710388352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380695:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776088.380696:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776088.380697:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776088.380697:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776088.380698:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776088.380699:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776088.380701:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776088.380702:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776088.380703:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776088.380706:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70000 x1862022710388352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776088.380711:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710388352:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776088.380713:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776088.380714:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776088.380715:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70e00 x1862022710388224/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380719:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776088.380719:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776088.380720:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7000. 02000000:00000001:0.0:1775776088.380722:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776088.380722:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776088.380723:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776088.380724:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776088.380725:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34000. 02000000:00000001:0.0:1775776088.380727:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776088.380728:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776088.380729:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70e00. 00000100:00000001:0.0:1775776088.380732:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776088.380733:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776088.380734:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776088.380737:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70000 x1862022710388352/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776113 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776088.380743:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776088.380744:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776088.380745:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776088.380747:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776088.380748:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776088.380750:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776088.380752:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776088.380753:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35c00. 02000000:00000001:0.0:1775776088.380755:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776088.380757:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776088.380758:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70000. 00000100:00000001:0.0:1775776088.380760:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776088.380761:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776088.380763:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776088.380766:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776088.380767:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776088.380769:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776088.380770:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776088.507247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776088.507251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776088.507257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776088.507258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776088.507261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776088.507262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776088.507336:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776088.507341:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776088.507345:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776089.403279:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776089.403287:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776089.403289:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776089.403296:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776089.403301:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776089.403306:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776089.403310:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776089.403312:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776089.403315:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776089.531240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776089.531246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776089.531252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776089.531255:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776089.531257:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776089.531261:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776089.531264:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776089.531266:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776089.531268:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776089.531270:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776089.531274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776089.531275:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776089.531276:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776089.531277:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776089.531278:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776089.531280:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776089.531281:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776089.531283:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776089.531358:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776089.531364:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776089.531369:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776090.427249:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776090.427259:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776090.427261:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776090.427268:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776090.427273:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776090.427276:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776090.427279:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776090.427281:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776090.427284:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776090.555234:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776090.555239:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776090.555242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776090.555242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776090.555245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776090.555246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776090.555362:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776090.555367:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776090.555370:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776091.579225:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776091.579231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776091.579235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776091.579236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776091.579239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776091.579240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776092.283230:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307647238, at 4307647296 00000100:00000010:0.0:1775776092.283243:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3480. 00010000:00000001:3.0:1775776092.603254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776092.603260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776092.603265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776092.603266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776092.603269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776092.603271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776093.499213:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776093.499220:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776093.499225:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776093.499228:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776093.499233:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776093.499240:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12976 00000400:00000200:3.0:1775776093.499249:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776093.499253:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776093.499256:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776093.499259:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776093.499261:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776093.499263:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776093.499264:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776093.499267:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776093.499268:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776093.499270:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776093.499271:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776093.499273:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776093.499274:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776093.499276:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776093.499277:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776093.499278:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776093.499279:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776093.499281:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776093.499284:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776093.499286:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776093.499288:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776093.499290:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776093.499292:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776093.499293:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776093.499294:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776093.499301:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73100. 00000020:00000040:3.0:1775776093.499303:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776093.499306:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776093.499307:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776093.499309:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776093.499311:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776093.499314:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776093.499327:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000100:00000001:3.0:1775776093.499331:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776093.499337:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73100 x1862022710388480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776093.499345:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776093.499348:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776093.499350:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f67e6552] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73100 x1862022710388480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776093.499365:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776093.499367:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776093.499369:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776093.499370:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776093.499371:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776093.499373:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776093.499374:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12976 00000400:00000200:3.0:1775776093.499376:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776093.499378:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776093.499383:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776093.499385:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776093.499387:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776093.499388:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776093.499390:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776093.499392:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776093.499392:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776093.499393:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776093.499395:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776093.499396:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776093.499397:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775776093.499399:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1775776093.499399:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776093.499400:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776093.499403:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776093.499404:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776093.499407:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776093.499408:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776093.499412:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000040:0.0:1775776093.499413:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73100 x1862022710388480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775776093.499417:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776093.499424:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776093.499425:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1775776093.499427:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775776093.499428:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776093.499432:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776093.499433:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776093.499434:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776093.499439:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776093.499439:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776093.499444:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00100000:0.0:1775776093.499446:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710388480:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776093.499449:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00100000:3.0:1775776093.499449:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1775776093.499452:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776093.499455:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775776093.499455:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776093.499459:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775776093.499460:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776093.499463:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 00000100:00000010:3.0:1775776093.499465:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 02000000:00000001:0.0:1775776093.499466:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776093.499471:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000400:00000200:0.0:1775776093.499474:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776093.499476:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000010:0.0:1775776093.499478:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 02000000:00000001:3.0:1775776093.499479:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776093.499483:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776093.499484:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:0.0:1775776093.499485:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 02000000:00000001:3.0:1775776093.499486:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775776093.499493:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710388480, portal 25 02000000:00000010:3.0:1775776093.499493:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000100:00000001:0.0:1775776093.499497:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776093.499499:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849016576 : -123823860535040 : ffff8f6202c73100) 00000100:00000001:3.0:1775776093.499500:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776093.499504:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73100 x1862022710388480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776093.499508:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710388608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776093.499515:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776093.499530:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776093.499534:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000001:3.0:1775776093.499536:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:0.0:1775776093.499538:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710388480, offset 0 00000100:00000001:3.0:1775776093.499544:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776093.499547:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776093.499550:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710388608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776093.499556:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776093.499561:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776093.499564:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775776093.499565:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00080000:3.0:1775776093.499569:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776093.499571:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1775776093.499572:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:3.0:1775776093.499573:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776093.499576:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:0.0:1775776093.499578:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00080000:3.0:1775776093.499580:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12976 00000400:00000200:0.0:1775776093.499585:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1775776093.499585:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776093.499590:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:3.0:1775776093.499591:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776093.499594:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1775776093.499598:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453587:453587:256:4294967295] 192.168.201.115@tcp LPNI seq info [453512:453512:8:4294967295] 00000100:00000040:3.0:1775776093.499599:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776093.499603:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775776093.499606:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775776093.499610:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776093.499613:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776093.499617:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776093.499619:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000400:00000200:0.0:1775776093.499620:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000020:00000001:3.0:1775776093.499621:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776093.499624:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776093.499626:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:0.0:1775776093.499630:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000020:00000001:3.0:1775776093.499631:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776093.499637:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000800:00000010:0.0:1775776093.499638:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000100:00000001:3.0:1775776093.499640:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776093.499644:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776093.499648:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000200:0.0:1775776093.499649:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000040:3.0:1775776093.499653:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000800:00000200:0.0:1775776093.499655:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00080000:3.0:1775776093.499659:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776093.499663:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775776093.499664:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:3.0:1775776093.499666:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:0.1:1775776093.499670:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 02000000:00000001:3.0:1775776093.499671:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776093.499676:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776093.499679:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000100:0.0:1775776093.499680:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000008:00100000:3.0:1775776093.499681:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776093.499684:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776093.499685:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776093.499687:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000800:00000010:0.0:1775776093.499689:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000020:00000040:3.0:1775776093.499690:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776093.499694:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776093.499695:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776093.499697:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1775776093.499697:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776093.499698:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776093.499700:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776093.499704:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 02000000:00000010:3.0:1775776093.499704:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000100:00000001:3.0:1775776093.499708:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776093.499709:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000100:00100000:3.0:1775776093.499713:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710388736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1775776093.499714:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775776093.499720:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776093.499725:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73100 x1862022710388480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776093.499725:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776093.499729:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776093.499737:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710388736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.499738:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776093.499743:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73100 x1862022710388480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776118 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776093.499759:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776093.499764:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776093.499768:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776093.499769:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1775776093.499770:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1775776093.499773:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776093.499775:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776093.499776:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:0.0:1775776093.499781:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775776093.499781:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12986) 00000100:00000001:0.0:1775776093.499783:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307652358 00000100:00000010:0.0:1775776093.499788:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776093.499793:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776093.499798:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307652358 00000100:00000010:0.0:1775776093.499800:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3720. 00000100:00000001:0.0:1775776093.499802:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776093.499804:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776093.499806:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776093.499807:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776093.499810:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776093/real 1775776093] req@ffff8f6202c73100 x1862022710388480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776093.499816:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776093.499817:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776093.499818:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775776093.499820:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776093.499821:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776093.499822:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776093.499824:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73100 x1862022710388480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776093.499829:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73100 x1862022710388480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.499833:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776093.499834:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776093.499835:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776093.499836:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776093.499838:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73100 x1862022710388480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.499843:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776093.499845:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73100 x1862022710388480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.499849:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776093.499851:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776093.499852:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776093.499858:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776093.499860:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776093.499861:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776093.499863:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776093.499865:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776093.499866:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776093.499869:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73100 x1862022710388480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776093.499873:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710388480:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776093.499876:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776093.499877:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776093.499879:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73100 x1862022710388480/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.499883:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776093.499884:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776093.499886:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776093.499887:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776093.499888:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776093.499889:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776093.499890:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776093.499892:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35c00. 02000000:00000001:0.0:1775776093.499894:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776093.499896:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776093.499897:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73100. 00000100:00000001:0.0:1775776093.499899:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776093.499900:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776093.499902:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776093.499906:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776093.499907:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776093.499908:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776093.499909:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776093.499910:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776093.499912:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70000 x1862022710388608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.499916:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776093.499917:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776093.499918:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776093.499920:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776093.499921:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776093.499922:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710388608:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776093.499924:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776093.499925:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776093.499926:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776093.499927:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776093.499928:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776093.499929:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776093.499932:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776093.499933:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775776093.499936:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776093.499937:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710388608, portal 10 00000100:00000001:0.0:1775776093.499939:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776093.499939:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004032 : -123823860547584 : ffff8f6202c70000) 00000100:00000040:0.0:1775776093.499943:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70000 x1862022710388608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.499947:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776093.499948:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776093.499950:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775776093.499951:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710388608, offset 0 00000400:00000200:0.0:1775776093.499954:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776093.499956:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776093.499958:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776093.499960:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776093.499962:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776093.499963:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776093.499965:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776093.499966:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453588:453588:256:4294967295] 192.168.201.115@tcp LPNI seq info [453513:453513:8:4294967295] 00000400:00000200:0.0:1775776093.499969:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776093.499973:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776093.499975:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776093.499977:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775776093.499979:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776093.499981:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775776093.499983:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776093.499986:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775776093.499988:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776093.499989:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775776093.499990:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776093.499992:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776093.499994:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775776093.499995:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775776093.499997:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776093.500000:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70000 x1862022710388608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.500007:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776093.500010:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70000 x1862022710388608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776118 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.500016:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776093.500017:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776093.500020:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776093.500024:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776093.500027:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776093.500028:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307648523 00000100:00000001:0.0:1775776093.500031:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307652358 00000100:00000001:0.0:1775776093.500033:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307648523 00000100:00000001:0.0:1775776093.500035:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307652358 00000100:00000001:0.0:1775776093.500037:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776093.500039:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776093.500041:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776093.500044:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776093/real 1775776093] req@ffff8f6202c70000 x1862022710388608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776093.500052:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776093.500053:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776093.500055:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775776093.500056:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776093.500057:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776093.500058:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776093.500060:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70000 x1862022710388608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776093.500065:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70000 x1862022710388608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.500069:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776093.500069:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776093.500070:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776093.500071:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776093.500073:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70000 x1862022710388608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.500077:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776093.500079:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70000 x1862022710388608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.500083:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776093.500085:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776093.500086:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776093.500087:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776093.500088:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776093.500091:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776093.500092:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776093.500094:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776093.500095:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776093.500097:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70000 x1862022710388608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776093.500102:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710388608:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776093.500104:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776093.500105:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70e00 x1862022710388736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.500109:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776093.500109:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776093.500111:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776093.500112:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776093.500113:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776093.500114:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710388736:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776093.500116:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776093.500117:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776093.500118:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776093.500119:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776093.500121:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7000. 02000000:00000001:0.0:1775776093.500122:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776093.500125:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776093.500126:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775776093.500128:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776093.500130:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710388736, portal 10 00000100:00000001:0.0:1775776093.500131:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776093.500131:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849007616 : -123823860544000 : ffff8f6202c70e00) 00000100:00000040:0.0:1775776093.500161:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70e00 x1862022710388736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.500167:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776093.500169:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776093.500174:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775776093.500176:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710388736, offset 0 00000400:00000200:0.0:1775776093.500179:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776093.500183:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776093.500187:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776093.500189:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776093.500192:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776093.500194:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776093.500196:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776093.500198:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453589:453589:256:4294967295] 192.168.201.115@tcp LPNI seq info [453514:453514:8:4294967295] 00000400:00000200:0.0:1775776093.500202:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776093.500209:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776093.500213:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776093.500216:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775776093.500219:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776093.500220:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775776093.500223:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776093.500225:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775776093.500227:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776093.500228:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775776093.500230:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776093.500232:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776093.500234:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775776093.500235:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775776093.500237:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776093.500239:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70e00 x1862022710388736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.500244:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776093.500245:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70e00 x1862022710388736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776118 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.500249:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776093.500250:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776093.500252:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776093.500253:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776093.500255:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776093.500257:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307648524 00000100:00000001:0.0:1775776093.500258:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307652358 00000100:00000001:0.0:1775776093.500259:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307648524 00000100:00000001:0.0:1775776093.500261:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307652358 00000100:00000001:0.0:1775776093.500262:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776093.500263:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776093.500264:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776093.500266:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776093/real 1775776093] req@ffff8f6202c70e00 x1862022710388736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776093.500270:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776093.500271:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776093.500272:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775776093.500273:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776093.500274:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776093.500275:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776093.500277:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70e00 x1862022710388736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776093.500282:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70e00 x1862022710388736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.500286:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776093.500287:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776093.500288:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776093.500288:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776093.500290:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70e00 x1862022710388736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.500293:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776093.500295:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70e00 x1862022710388736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.500307:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776093.500309:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776093.500309:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776093.500310:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776093.500311:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776093.500312:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776093.500314:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776093.500315:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776093.500316:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776093.500319:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70e00 x1862022710388736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776093.500323:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710388736:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776093.500325:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776093.500326:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776093.500328:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70000 x1862022710388608/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.500331:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776093.500333:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776093.500334:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776093.500335:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776093.500336:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776093.500337:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776093.500338:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776093.500339:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34000. 02000000:00000001:0.0:1775776093.500341:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776093.500342:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776093.500343:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70000. 00000100:00000001:0.0:1775776093.500344:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776093.500345:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776093.500346:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776093.500347:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70e00 x1862022710388736/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776118 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776093.500351:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776093.500351:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776093.500352:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7000. 02000000:00000001:0.0:1775776093.500353:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776093.500354:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776093.500354:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776093.500355:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776093.500356:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36c00. 02000000:00000001:0.0:1775776093.500357:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776093.500358:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776093.500358:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70e00. 00000100:00000001:0.0:1775776093.500359:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776093.500360:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776093.500361:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776093.500363:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776093.500364:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776093.500365:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776093.500366:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776093.627203:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776093.627207:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776093.627211:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776093.627211:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776093.627214:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776093.627215:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776093.627263:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776093.627266:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776093.627268:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776094.651230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776094.651237:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776094.651239:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776094.651242:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776094.651246:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776094.651248:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776094.651250:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776094.651252:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776094.651256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776094.651257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776094.651261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776094.651262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776094.651318:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776094.651322:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776094.651325:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776095.675219:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776095.675226:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776095.675231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776095.675233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776095.675237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776095.675246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776095.675310:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776095.675315:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776095.675318:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776096.699250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776096.699256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776096.699261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776096.699262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776096.699265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776096.699266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776097.403240:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307652358, at 4307652416 00000100:00000010:0.0:1775776097.403260:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3480. 00010000:00000001:3.0:1775776097.723276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776097.723284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776097.723292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776097.723293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776097.723298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776097.723299:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776098.619361:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776098.619370:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776098.619376:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776098.619387:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776098.619395:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776098.619404:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776098.619410:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776098.619413:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776098.619417:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776098.619596:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776098.619602:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776098.619605:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776098.619617:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491aa00 x1862022710388864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.619630:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776098.619632:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776098.619635:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776098.619639:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776098.619642:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776098.619647:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710388864:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776098.619653:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776098.619658:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776098.619660:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776098.619671:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776098.619677:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776098.619680:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776098.619689:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776098.619695:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775776098.619703:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776098.619712:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710388864, portal 25 00000100:00000001:0.0:1775776098.619716:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776098.619718:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684369408 : -123823025182208 : ffff8f623491aa00) 00000100:00000040:0.0:1775776098.619726:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491aa00 x1862022710388864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.619745:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776098.619750:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776098.619753:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775776098.619758:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710388864, offset 0 00000400:00000200:0.0:1775776098.619767:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776098.619776:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776098.619783:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776098.619789:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776098.619794:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776098.619801:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776098.619804:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776098.619809:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453590:453590:256:4294967295] 192.168.201.115@tcp LPNI seq info [453515:453515:8:4294967295] 00000400:00000200:0.0:1775776098.619815:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776098.619830:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776098.619839:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776098.619848:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775776098.619862:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776098.619869:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775776098.619881:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776098.619885:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775776098.619890:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776098.619894:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775776098.619900:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776098.619907:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776098.619916:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775776098.619919:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775776098.619923:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776098.619928:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491aa00 x1862022710388864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.619939:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776098.619943:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491aa00 x1862022710388864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776123 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.619950:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776098.619953:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776098.619960:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776098.619965:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776098.619968:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307657478 00000100:00000010:0.0:1775776098.619973:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776098.619981:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776098.619988:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307657478 00000100:00000010:0.0:1775776098.619992:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c30c0. 00000100:00000001:0.0:1775776098.619995:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776098.619999:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776098.620001:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776098.620004:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776098.620008:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776098/real 1775776098] req@ffff8f623491aa00 x1862022710388864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776098.620019:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776098.620021:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776098.620023:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775776098.620026:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776098.620027:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776098.620029:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776098.620033:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491aa00 x1862022710388864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776098.620042:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491aa00 x1862022710388864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.620049:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776098.620051:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776098.620053:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776098.620056:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776098.620059:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491aa00 x1862022710388864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.620066:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776098.620071:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491aa00 x1862022710388864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.620076:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776098.620078:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776098.620079:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776098.620079:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776098.620081:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776098.620083:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776098.620086:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776098.620088:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776098.620090:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776098.620094:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491aa00 x1862022710388864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776098.620102:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710388864:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776098.620107:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776098.620110:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776098.620113:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491aa00 x1862022710388864/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.620120:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776098.620123:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776098.620126:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776098.620129:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776098.620131:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776098.620160:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776098.620164:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776098.620166:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4dc00. 02000000:00000001:0.0:1775776098.620171:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776098.620174:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776098.620176:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491aa00. 00000100:00000001:0.0:1775776098.620179:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776098.620180:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776098.620182:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776098.620190:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776098.620192:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776098.620194:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776098.620195:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776098.620197:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776098.620201:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918e00 x1862022710388992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.620212:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776098.620213:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776098.620215:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776098.620216:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776098.620217:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776098.620219:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710388992:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776098.620221:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776098.620223:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776098.620224:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776098.620225:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776098.620227:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776098.620228:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776098.620231:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776098.620233:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775776098.620236:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776098.620238:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710388992, portal 10 00000100:00000001:0.0:1775776098.620239:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776098.620240:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684362240 : -123823025189376 : ffff8f6234918e00) 00000100:00000040:0.0:1775776098.620245:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918e00 x1862022710388992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.620249:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776098.620251:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776098.620252:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775776098.620254:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710388992, offset 0 00000400:00000200:0.0:1775776098.620256:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776098.620258:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776098.620260:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776098.620262:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776098.620264:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776098.620266:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776098.620267:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776098.620270:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453591:453591:256:4294967295] 192.168.201.115@tcp LPNI seq info [453516:453516:8:4294967295] 00000400:00000200:0.0:1775776098.620272:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776098.620276:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776098.620279:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776098.620281:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775776098.620284:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776098.620286:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775776098.620288:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776098.620290:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775776098.620292:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776098.620294:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775776098.620295:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776098.620298:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776098.620300:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775776098.620301:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775776098.620302:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776098.620304:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918e00 x1862022710388992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.620309:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776098.620311:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918e00 x1862022710388992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776123 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.620315:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776098.620316:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776098.620318:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776098.620321:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776098.620323:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776098.620324:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307653644 00000100:00000001:0.0:1775776098.620326:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307657478 00000100:00000001:0.0:1775776098.620327:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307653644 00000100:00000001:0.0:1775776098.620329:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307657478 00000100:00000001:0.0:1775776098.620330:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776098.620332:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776098.620333:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776098.620335:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776098/real 1775776098] req@ffff8f6234918e00 x1862022710388992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776098.620339:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776098.620340:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776098.620341:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775776098.620342:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776098.620343:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776098.620344:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776098.620346:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918e00 x1862022710388992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776098.620351:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918e00 x1862022710388992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.620354:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776098.620355:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776098.620356:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776098.620356:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776098.620358:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918e00 x1862022710388992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.620362:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776098.620363:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918e00 x1862022710388992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.620367:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776098.620368:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776098.620369:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776098.620370:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776098.620371:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776098.620375:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776098.620376:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776098.620379:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776098.620380:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776098.620383:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918e00 x1862022710388992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776098.620387:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710388992:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776098.620389:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776098.620391:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918380 x1862022710389120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.620394:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776098.620395:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776098.620396:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776098.620396:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776098.620398:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776098.620399:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710389120:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776098.620401:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776098.620402:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776098.620403:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776098.620404:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776098.620406:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7000. 02000000:00000001:0.0:1775776098.620407:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776098.620409:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776098.620410:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775776098.620411:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776098.620413:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710389120, portal 10 00000100:00000001:0.0:1775776098.620414:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776098.620415:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684359552 : -123823025192064 : ffff8f6234918380) 00000100:00000040:0.0:1775776098.620418:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918380 x1862022710389120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.620422:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776098.620424:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776098.620425:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775776098.620427:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710389120, offset 0 00000400:00000200:0.0:1775776098.620429:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776098.620444:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776098.620447:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776098.620449:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776098.620450:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776098.620452:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776098.620453:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776098.620455:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453592:453592:256:4294967295] 192.168.201.115@tcp LPNI seq info [453517:453517:8:4294967295] 00000400:00000200:0.0:1775776098.620458:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776098.620461:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776098.620463:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776098.620465:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000800:00000200:0.0:1775776098.620468:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776098.620470:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775776098.620472:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776098.620473:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775776098.620476:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776098.620477:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:0.0:1775776098.620479:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776098.620481:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776098.620483:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:0.0:1775776098.620483:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:0.0:1775776098.620485:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776098.620487:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918380 x1862022710389120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.620492:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776098.620494:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918380 x1862022710389120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776123 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.620498:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776098.620499:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776098.620501:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776098.620503:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776098.620505:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776098.620506:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307653644 00000100:00000001:0.0:1775776098.620507:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307657478 00000100:00000001:0.0:1775776098.620509:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307653644 00000100:00000001:0.0:1775776098.620510:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307657478 00000100:00000001:0.0:1775776098.620511:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776098.620512:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776098.620513:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776098.620515:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776098/real 1775776098] req@ffff8f6234918380 x1862022710389120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776098.620519:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776098.620520:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776098.620521:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775776098.620522:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776098.620523:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776098.620524:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776098.620526:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918380 x1862022710389120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776098.620530:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918380 x1862022710389120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.620533:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776098.620534:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776098.620535:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776098.620536:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776098.620538:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918380 x1862022710389120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.620541:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776098.620543:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918380 x1862022710389120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.620548:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776098.620549:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776098.620550:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776098.620551:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776098.620551:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776098.620552:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776098.620554:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776098.620555:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776098.620556:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776098.620558:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918380 x1862022710389120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776098.620562:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710389120:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776098.620564:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776098.620565:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776098.620567:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918e00 x1862022710388992/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.620571:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776098.620572:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776098.620573:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776098.620575:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776098.620576:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776098.620577:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776098.620578:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776098.620579:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:0.0:1775776098.620581:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776098.620582:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776098.620582:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918e00. 00000100:00000001:0.0:1775776098.620584:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776098.620585:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776098.620587:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776098.620588:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918380 x1862022710389120/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776123 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776098.620592:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776098.620593:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776098.620593:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7000. 02000000:00000001:0.0:1775776098.620595:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776098.620595:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776098.620596:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776098.620597:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776098.620597:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4c000. 02000000:00000001:0.0:1775776098.620598:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776098.620599:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776098.620599:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918380. 00000100:00000001:0.0:1775776098.620600:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776098.620601:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776098.620602:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776098.620604:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776098.620605:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776098.620606:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776098.620606:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776098.747239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776098.747246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776098.747254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776098.747255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776098.747260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776098.747261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776098.747322:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776098.747328:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776098.747331:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776099.643224:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776099.643233:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776099.643235:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776099.643242:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776099.643245:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776099.643249:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776099.643251:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776099.643254:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776099.643256:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776099.771278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776099.771287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776099.771293:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776099.771297:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776099.771300:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776099.771304:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776099.771306:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776099.771309:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776099.771311:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776099.771314:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776099.771317:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776099.771319:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776099.771320:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776099.771321:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776099.771323:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776099.771324:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776099.771326:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776099.771327:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776099.771392:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776099.771396:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776099.771433:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776100.667291:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776100.667300:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776100.667303:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776100.667310:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776100.667315:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776100.667320:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776100.667323:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776100.667325:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776100.667328:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776100.795213:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776100.795219:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776100.795223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776100.795224:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776100.795226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776100.795227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776100.795277:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776100.795281:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776100.795284:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776101.819274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776101.819280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776101.819285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776101.819285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776101.819288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776101.819289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776102.523251:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307657478, at 4307657536 00000100:00000010:0.0:1775776102.523271:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3480. 00010000:00000001:3.0:1775776102.843275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776102.843282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776102.843287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776102.843288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776102.843291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776102.843291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776103.739259:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776103.739268:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776103.739273:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776103.739277:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776103.739281:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776103.739289:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12986 00000400:00000200:0.0:1775776103.739299:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776103.739304:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776103.739307:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776103.739310:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776103.739314:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776103.739317:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776103.739320:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776103.739323:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776103.739325:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776103.739329:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776103.739332:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776103.739334:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776103.739336:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776103.739339:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776103.739353:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776103.739356:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776103.739358:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776103.739361:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776103.739365:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776103.739367:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776103.739371:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776103.739375:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776103.739378:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776103.739380:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776103.739382:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776103.739393:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061f80. 00000020:00000040:0.0:1775776103.739397:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776103.739401:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776103.739402:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776103.739405:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776103.739407:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776103.739411:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776103.739420:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7000. 00000100:00000001:0.0:1775776103.739426:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776103.739434:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061f80 x1862022710389248/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776103.739446:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776103.739450:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776103.739453:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000029d3794] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061f80 x1862022710389248/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776103.739472:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776103.739475:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776103.739477:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776103.739479:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776103.739480:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776103.739483:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776103.739485:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12986 00000400:00000200:0.0:1775776103.739488:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776103.739492:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776103.739494:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776103.739498:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776103.739500:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776103.739504:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776103.739506:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776103.739510:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776103.739512:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776103.739515:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776103.739518:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775776103.739521:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775776103.739525:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775776103.739528:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776103.739531:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776103.739532:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776103.739533:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776103.739538:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775776103.739540:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776103.739543:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776103.739546:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776103.739551:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776103.739554:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776103.739556:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776103.739560:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776103.739563:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776103.739567:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776103.739569:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063b80. 00000020:00000040:0.0:1775776103.739572:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776103.739573:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776103.739574:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776103.739575:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776103.739577:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776103.739579:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776103.739583:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5800. 00000100:00000001:0.0:1775776103.739587:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776103.739593:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063b80 x1862022710389376/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776103.739600:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776103.739603:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776103.739607:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ee9007b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063b80 x1862022710389376/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776103.739615:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776103.739618:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776103.739621:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776103.739623:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776103.739625:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776103.739630:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776103.739635:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12986 00000400:00000200:0.0:1775776103.739642:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776103.739647:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776103.739649:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776103.739653:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776103.739655:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776103.739659:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776103.739662:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776103.739663:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776103.739664:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776103.739665:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776103.739666:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775776103.739667:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775776103.739672:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775776103.739675:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776103.739676:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776103.739681:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776103.739683:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776103.739685:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775776103.739689:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776103.739691:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776103.739693:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776103.739697:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776103.739701:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776103.739703:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776103.739704:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776103.739705:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776103.739706:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776103.739708:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061500. 00000020:00000040:0.0:1775776103.739711:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776103.739714:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776103.739718:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776103.739721:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776103.739724:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776103.739727:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776103.739730:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6c00. 00000100:00000001:0.0:1775776103.739732:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776103.739737:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061500 x1862022710389504/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776103.739751:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776103.739753:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776103.739756:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb8d992] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061500 x1862022710389504/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776103.739766:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776103.739771:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776103.739773:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776103.739775:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776103.739780:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776103.739785:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (12996) 00000800:00000200:1.1:1775776103.739790:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776103.739793:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775776103.740158:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776103.740161:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775776103.740372:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776103.740373:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00010000:00000001:3.0:1775776103.867276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776103.867284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776103.867291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776103.867293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776103.867296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776103.867298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776103.867378:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776103.867383:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776103.867386:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.0:1775776104.571225:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775776104.571232:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775776104.571236:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1775776104.571417:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776104.571437:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775776104.571443:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776104.571447:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1775776104.891263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776104.891274:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776104.891277:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776104.891281:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776104.891287:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776104.891290:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776104.891292:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776104.891295:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776104.891300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776104.891301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776104.891304:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776104.891305:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776104.891369:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776104.891373:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776104.891376:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776105.915264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776105.915271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776105.915276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776105.915277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776105.915279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776105.915280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776105.915344:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776105.915348:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776105.915351:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776106.939237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776106.939241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776106.939245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776106.939245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776106.939248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776106.939249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776107.643268:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307662598, at 4307662656 00000100:00000010:0.0:1775776107.643293:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2de0. 00010000:00000001:3.0:1775776107.963234:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776107.963237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776107.963241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776107.963242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776107.963244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776107.963245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776108.859245:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776108.859265:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776108.859267:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776108.859272:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776108.859276:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776108.859284:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12991 00000400:00000200:2.0:1775776108.859295:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776108.859303:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776108.859305:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776108.859308:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776108.859312:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776108.859314:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776108.859316:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776108.859318:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776108.859320:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776108.859322:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776108.859324:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776108.859326:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776108.859328:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776108.859330:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776108.859331:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776108.859333:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776108.859334:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776108.859336:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776108.859338:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776108.859339:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776108.859342:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776108.859344:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776108.859347:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776108.859350:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776108.859352:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776108.859361:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775776108.859364:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776108.859367:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776108.859368:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776108.859371:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776108.859373:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776108.859377:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776108.859384:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775776108.859390:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776108.859399:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710389632/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776108.859406:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776108.859410:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776108.859413:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710389632/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776108.859428:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776108.859431:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776108.859433:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776108.859434:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776108.859436:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776108.859438:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776108.859441:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12991 00000400:00000200:2.0:1775776108.859443:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776108.859446:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776108.859447:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776108.859451:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776108.859454:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776108.859457:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776108.859459:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776108.859462:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776108.859464:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776108.859467:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776108.859469:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776108.859473:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776108.859476:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776108.859480:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776108.859482:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776108.859484:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776108.859486:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776108.859491:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776108.859495:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776108.859498:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776108.859502:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776108.859508:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776108.859510:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776108.859513:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776108.859518:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776108.859521:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776108.859525:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776108.859530:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775776108.859534:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776108.859536:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776108.859537:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776108.859539:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776108.859541:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776108.859542:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776108.859545:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775776108.859548:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776108.859552:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710389760/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776108.859559:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776108.859562:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776108.859567:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710389760/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776108.859577:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776108.859579:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776108.859586:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776108.859588:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776108.859591:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776108.859594:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776108.859598:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12991 00000400:00000200:2.0:1775776108.859603:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776108.859607:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776108.859612:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776108.859614:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776108.859615:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776108.859619:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776108.859621:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776108.859624:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776108.859624:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776108.859625:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776108.859627:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776108.859628:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776108.859629:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776108.859631:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776108.859633:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776108.859635:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776108.859639:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776108.859640:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776108.859642:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776108.859645:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776108.859646:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776108.859648:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776108.859651:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776108.859652:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776108.859654:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776108.859656:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776108.859659:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776108.859661:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775776108.859664:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776108.859665:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776108.859666:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776108.859667:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776108.859668:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776108.859668:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776108.859673:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775776108.859676:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776108.859682:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710389888/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776108.859697:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776108.859700:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776108.859703:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710389888/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776108.859717:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776108.859720:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776108.859725:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776108.859728:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776108.859730:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776108.859743:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13001) 00000800:00000200:1.1:1775776108.859749:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776108.859752:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775776108.860024:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776108.860026:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775776108.860210:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776108.860211:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00010000:00000001:3.0:1775776108.987306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776108.987314:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776108.987321:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776108.987322:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776108.987326:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776108.987328:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776108.987386:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776108.987389:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776108.987392:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776110.011282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776110.011290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776110.011296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776110.011299:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776110.011301:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776110.011304:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776110.011307:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776110.011309:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776110.011310:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776110.011312:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776110.011315:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776110.011316:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776110.011317:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776110.011318:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776110.011318:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776110.011319:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776110.011320:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776110.011321:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776110.011385:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776110.011389:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776110.011392:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776111.035260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776111.035268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776111.035274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776111.035276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776111.035280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776111.035281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776111.035338:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776111.035342:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776111.035344:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776112.059248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776112.059257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776112.059264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776112.059265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776112.059270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776112.059272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776112.763284:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307667718, at 4307667776 00000100:00000010:0.0:1775776112.763299:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2660. 00010000:00000001:3.0:1775776113.083204:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776113.083214:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776113.083220:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776113.083221:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776113.083223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776113.083224:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776113.979290:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776113.979301:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776113.979304:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776113.979309:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776113.979320:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776113.979328:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 12996 00000400:00000200:3.0:1775776113.979337:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776113.979349:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776113.979352:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776113.979361:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776113.979368:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776113.979370:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776113.979373:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776113.979377:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776113.979378:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776113.979381:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776113.979383:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776113.979385:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776113.979387:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776113.979389:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776113.979391:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776113.979393:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776113.979394:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776113.979396:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776113.979399:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776113.979401:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776113.979403:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776113.979406:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776113.979408:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776113.979410:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776113.979411:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776113.979419:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775776113.979422:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776113.979425:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776113.979426:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776113.979428:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776113.979430:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776113.979434:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776113.979440:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000001:3.0:1775776113.979445:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776113.979453:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710390016/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776113.979460:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776113.979465:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776113.979470:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710390016/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776113.979495:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776113.979499:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776113.979503:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776113.979505:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776113.979507:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776113.979510:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776113.979513:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12996 00000400:00000200:3.0:1775776113.979518:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776113.979522:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776113.979525:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776113.979529:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776113.979531:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776113.979534:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776113.979537:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776113.979542:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776113.979544:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776113.979547:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776113.979551:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776113.979556:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776113.979561:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776113.979568:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776113.979574:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776113.979580:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776113.979585:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776113.979590:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776113.979596:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776113.979598:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776113.979601:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776113.979607:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776113.979612:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776113.979614:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776113.979620:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776113.979624:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776113.979626:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776113.979630:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775776113.979633:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776113.979637:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776113.979639:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776113.979643:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776113.979645:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776113.979648:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776113.979653:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000100:00000001:3.0:1775776113.979659:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776113.979665:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710390144/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776113.979678:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776113.979680:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776113.979684:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710390144/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776113.979695:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776113.979698:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776113.979702:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776113.979704:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776113.979707:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776113.979710:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776113.979714:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 12996 00000400:00000200:3.0:1775776113.979719:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776113.979727:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776113.979731:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776113.979734:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776113.979737:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776113.979741:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776113.979745:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776113.979747:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776113.979750:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776113.979753:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776113.979755:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776113.979756:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776113.979758:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776113.979763:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776113.979768:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776113.979773:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776113.979778:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776113.979779:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776113.979785:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776113.979789:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776113.979790:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776113.979792:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776113.979793:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776113.979797:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776113.979799:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776113.979800:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776113.979801:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776113.979804:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775776113.979807:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776113.979811:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776113.979812:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776113.979816:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.1:1775776113.979816:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:3.0:1775776113.979818:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776113.979821:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776113.979824:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:3.0:1775776113.979826:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776113.979829:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710390272/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776113.979834:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776113.979835:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776113.979841:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710390272/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776113.979852:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776113.979856:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776113.979859:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776113.979862:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776113.979865:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776113.979869:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13006) 00000800:00000200:1.1:1775776113.980024:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776113.980200:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00010000:00000001:3.0:1775776114.107266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776114.107270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776114.107274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776114.107275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776114.107278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776114.107279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776114.107353:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776114.107357:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776114.107361:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776115.131275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776115.131281:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776115.131284:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776115.131287:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776115.131292:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776115.131294:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776115.131297:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776115.131299:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776115.131303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776115.131305:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776115.131309:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776115.131310:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776115.131388:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776115.131392:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776115.131395:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776116.155256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776116.155263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776116.155270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776116.155271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776116.155275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776116.155277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776116.155345:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776116.155349:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776116.155352:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775776117.051394:0:2400:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:2.0:1775776117.051428:0:2400:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1775776117.051439:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776117.051443:0:2400:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:2.0:1775776117.051449:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776117.051453:0:2400:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:2.0:1775776117.051458:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775776117.051462:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:2.0:1775776117.051467:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775776117.051472:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775776117.051479:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3880 00000400:00000010:2.0:1775776117.051482:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3880. 00000100:00000001:2.0:1775776117.051488:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776117.051497:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c70380 x1862022710390016/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776117.051527:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776117.051533:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70380 x1862022710390016/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776138 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776117.051542:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776117.051544:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775776117.051554:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775776117.051557:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230700. 00000400:00000200:2.0:1775776117.051559:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775776117.051563:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775776117.051566:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:2.0:1775776117.051568:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:2.0:1775776117.051571:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776117.051575:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c71c00 x1862022710390144/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776117.051584:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776117.051587:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71c00 x1862022710390144/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776138 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776117.051593:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776117.051594:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775776117.051599:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775776117.051601:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230a00. 00000400:00000200:2.0:1775776117.051604:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775776117.051607:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775776117.051609:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3ee0 00000400:00000010:2.0:1775776117.051611:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3ee0. 00000100:00000001:2.0:1775776117.051613:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776117.051617:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c73480 x1862022710390272/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776138 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776117.051625:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776117.051628:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73480 x1862022710390272/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776138 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776117.051635:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776117.051637:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:3.0:1775776117.179274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776117.179280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776117.179287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776117.179288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776117.179292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776117.179293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776117.883201:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307672838, at 4307672896 00000100:00000010:0.0:1775776117.883214:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2b40. 00010000:00000001:3.0:1775776118.203241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776118.203246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776118.203250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776118.203251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776118.203259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776118.203261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776119.099313:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776119.099318:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776119.099322:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776119.099325:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776119.099329:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776119.099335:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13001 00000400:00000200:3.0:1775776119.099344:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776119.099350:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776119.099353:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776119.099356:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776119.099359:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776119.099361:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776119.099363:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776119.099367:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776119.099368:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776119.099371:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776119.099373:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776119.099376:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776119.099378:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776119.099380:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776119.099382:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776119.099384:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776119.099385:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776119.099388:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776119.099391:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776119.099393:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776119.099397:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776119.099400:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776119.099403:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776119.099405:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776119.099408:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776119.099413:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775776119.099417:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776119.099420:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776119.099422:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776119.099425:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776119.099433:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776119.099436:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776119.099443:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000001:3.0:1775776119.099449:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776119.099457:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710390400/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776119.099467:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776119.099471:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776119.099476:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710390400/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776119.099497:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776119.099500:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776119.099503:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776119.099505:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776119.099506:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776119.099509:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776119.099511:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13001 00000400:00000200:3.0:1775776119.099514:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776119.099518:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776119.099520:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776119.099523:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776119.099526:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776119.099527:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776119.099530:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776119.099534:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776119.099537:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776119.099539:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776119.099541:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776119.099545:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776119.099547:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776119.099554:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776119.099559:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776119.099563:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776119.099567:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776119.099571:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776119.099575:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776119.099581:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776119.099584:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776119.099591:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776119.099595:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776119.099598:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776119.099603:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776119.099607:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776119.099609:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776119.099612:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000020:00000040:3.0:1775776119.099614:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776119.099616:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776119.099617:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776119.099619:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776119.099620:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776119.099622:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776119.099625:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:3.0:1775776119.099628:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776119.099633:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710390528/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776119.099640:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776119.099643:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776119.099646:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710390528/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776119.099654:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776119.099656:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776119.099666:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776119.099667:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776119.099669:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776119.099671:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776119.099673:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13001 00000400:00000200:3.0:1775776119.099678:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776119.099682:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776119.099684:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776119.099687:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776119.099692:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776119.099695:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776119.099697:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776119.099699:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776119.099701:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776119.099703:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776119.099706:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776119.099708:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776119.099712:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776119.099716:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776119.099718:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776119.099722:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776119.099725:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776119.099726:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776119.099729:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776119.099734:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776119.099736:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776119.099742:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776119.099745:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776119.099748:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776119.099751:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776119.099756:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776119.099760:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776119.099765:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775776119.099769:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776119.099772:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776119.099775:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776119.099777:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776119.099780:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776119.099782:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776119.099786:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775776119.099792:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776119.099799:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710390656/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776119.099813:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776119.099817:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776119.099824:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710390656/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776119.099840:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776119.099845:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776119.099852:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776119.099856:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776119.099863:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776119.099870:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13011) 00000800:00000200:1.1:1775776119.099898:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776119.099902:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775776119.100356:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776119.100358:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775776119.100632:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776119.100635:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00010000:00000001:3.0:1775776119.227247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776119.227251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776119.227254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776119.227255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776119.227258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776119.227259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776119.227311:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776119.227314:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776119.227317:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776120.251227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776120.251232:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776120.251238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776120.251241:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776120.251243:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776120.251246:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776120.251249:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776120.251251:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776120.251253:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776120.251255:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776120.251259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776120.251260:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776120.251261:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776120.251263:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776120.251264:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776120.251265:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776120.251267:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776120.251269:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776120.251324:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776120.251328:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776120.251331:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776121.275249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776121.275255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776121.275261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776121.275263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776121.275267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776121.275269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776121.275288:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776121.275293:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776121.275296:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776122.299192:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776122.299196:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776122.299200:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776122.299201:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776122.299203:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776122.299204:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776123.003216:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307677958, at 4307678016 00000100:00000010:0.0:1775776123.003232:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e20c0. 00010000:00000001:3.0:1775776123.323263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776123.323269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776123.323275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776123.323276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776123.323280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776123.323281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776124.219219:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776124.219229:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776124.219234:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776124.219238:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776124.219242:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776124.219249:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13006 00000400:00000200:3.0:1775776124.219256:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776124.219261:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776124.219264:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776124.219267:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776124.219269:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776124.219271:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776124.219272:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776124.219274:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776124.219275:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776124.219277:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776124.219279:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776124.219281:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776124.219283:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776124.219286:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776124.219288:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776124.219289:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776124.219291:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776124.219294:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776124.219297:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776124.219299:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776124.219302:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776124.219306:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776124.219309:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776124.219312:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776124.219313:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776124.219322:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775776124.219326:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776124.219330:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776124.219332:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776124.219333:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776124.219335:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776124.219338:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776124.219345:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000001:3.0:1775776124.219350:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776124.219356:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710390784/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776124.219363:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776124.219366:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776124.219369:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710390784/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776124.219385:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776124.219388:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776124.219391:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776124.219393:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776124.219394:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776124.219397:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776124.219400:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13006 00000400:00000200:3.0:1775776124.219404:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776124.219410:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776124.219414:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776124.219419:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776124.219424:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776124.219429:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776124.219434:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776124.219439:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776124.219440:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776124.219443:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776124.219445:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776124.219447:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776124.219448:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776124.219451:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776124.219452:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776124.219456:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776124.219460:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776124.219462:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776124.219466:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776124.219469:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776124.219471:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776124.219475:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776124.219478:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776124.219480:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776124.219483:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776124.219486:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776124.219487:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776124.219490:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775776124.219491:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776124.219493:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776124.219495:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776124.219496:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776124.219497:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776124.219499:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776124.219505:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000001:3.0:1775776124.219510:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776124.219515:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710390912/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776124.219523:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776124.219526:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776124.219531:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710390912/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776124.219548:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776124.219551:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776124.219556:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776124.219558:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776124.219560:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776124.219563:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776124.219566:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13006 00000400:00000200:3.0:1775776124.219572:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776124.219574:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776124.219578:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776124.219582:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776124.219584:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776124.219588:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776124.219590:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776124.219592:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776124.219594:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776124.219596:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776124.219598:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776124.219601:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776124.219604:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776124.219608:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776124.219610:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776124.219614:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776124.219617:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776124.219622:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776124.219625:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776124.219627:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776124.219629:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776124.219632:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776124.219634:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776124.219638:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776124.219639:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776124.219643:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776124.219647:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776124.219650:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775776124.219653:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776124.219655:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776124.219655:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776124.219659:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776124.219661:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776124.219663:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776124.219667:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000100:00000001:3.0:1775776124.219671:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776124.219671:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776124.219675:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000100:00100000:3.0:1775776124.219677:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710391040/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776124.219689:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776124.219690:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776124.219693:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710391040/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776124.219705:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776124.219708:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776124.219712:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776124.219714:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776124.219717:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776124.219722:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13016) 00000800:00000200:1.1:1775776124.219932:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776124.219933:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775776124.220119:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776124.220120:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00010000:00000001:3.0:1775776124.347213:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776124.347219:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776124.347222:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776124.347225:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776124.347229:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776124.347231:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776124.347233:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776124.347235:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776124.347239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776124.347241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776124.347244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776124.347246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776124.347292:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776124.347297:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776124.347300:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776125.371258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776125.371264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776125.371270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776125.371271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776125.371274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776125.371275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776125.371331:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776125.371334:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776125.371336:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776126.395246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776126.395252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776126.395255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776126.395256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776126.395260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776126.395261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776126.395328:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776126.395332:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776126.395335:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776127.419243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776127.419247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776127.419251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776127.419251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776127.419254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776127.419255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776128.123230:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307683078, at 4307683136 00000100:00000010:0.0:1775776128.123248:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ba0. 00010000:00000001:3.0:1775776128.443245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776128.443250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776128.443253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776128.443254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776128.443257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776128.443258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776129.339260:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776129.339269:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776129.339272:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776129.339275:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776129.339279:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776129.339285:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13011 00000400:00000200:0.0:1775776129.339294:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776129.339300:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776129.339303:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776129.339306:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776129.339308:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776129.339310:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776129.339312:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776129.339315:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776129.339317:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776129.339320:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776129.339322:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776129.339325:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776129.339327:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776129.339330:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776129.339331:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776129.339333:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776129.339334:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776129.339336:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776129.339338:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776129.339340:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776129.339342:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776129.339345:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776129.339347:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776129.339348:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776129.339349:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776129.339357:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062a00. 00000020:00000040:0.0:1775776129.339360:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776129.339363:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776129.339363:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776129.339365:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776129.339367:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776129.339370:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776129.339377:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7800. 00000100:00000001:0.0:1775776129.339381:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776129.339388:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062a00 x1862022710391168/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776129.339395:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776129.339398:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776129.339401:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009753f9f6] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062a00 x1862022710391168/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776129.339417:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776129.339420:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776129.339424:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776129.339425:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776129.339426:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776129.339429:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776129.339431:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13011 00000400:00000200:0.0:1775776129.339435:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776129.339437:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776129.339440:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776129.339443:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776129.339445:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776129.339447:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776129.339449:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776129.339451:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776129.339452:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776129.339454:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776129.339456:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775776129.339458:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775776129.339460:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775776129.339462:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776129.339464:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776129.339465:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776129.339467:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776129.339470:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775776129.339472:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776129.339475:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776129.339476:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776129.339480:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776129.339482:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776129.339484:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776129.339488:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776129.339490:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776129.339492:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776129.339495:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062d80. 00000020:00000040:0.0:1775776129.339496:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776129.339499:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776129.339499:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776129.339501:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776129.339502:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776129.339503:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776129.339506:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6000. 00000100:00000001:0.0:1775776129.339509:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776129.339513:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062d80 x1862022710391296/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776129.339521:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776129.339523:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776129.339526:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009965f80b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062d80 x1862022710391296/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776129.339534:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776129.339536:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776129.339540:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776129.339541:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776129.339542:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776129.339545:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776129.339546:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13011 00000400:00000200:0.0:1775776129.339586:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776129.339588:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776129.339590:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776129.339593:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776129.339595:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776129.339597:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776129.339599:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776129.339601:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776129.339602:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776129.339603:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776129.339605:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775776129.339607:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:1.1:1775776129.339607:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:0.0:1775776129.339609:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:1.1:1775776129.339610:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000100:00000040:0.0:1775776129.339612:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776129.339614:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776129.339616:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776129.339618:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776129.339620:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775776129.339622:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776129.339625:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776129.339626:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776129.339628:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776129.339630:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776129.339632:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776129.339633:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776129.339635:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776129.339637:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776129.339639:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062680. 00000020:00000040:0.0:1775776129.339641:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776129.339643:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776129.339644:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776129.339646:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776129.339647:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776129.339649:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776129.339652:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000001:0.0:1775776129.339654:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776129.339658:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062680 x1862022710391424/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776129.339665:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776129.339666:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776129.339669:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000082c7985e] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062680 x1862022710391424/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776129.339676:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776129.339679:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776129.339682:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776129.339685:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776129.339688:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776129.339692:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13022) 00000800:00000200:1.1:1775776129.339879:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776129.339880:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1775776129.340055:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776129.340057:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00010000:00000001:3.0:1775776129.467267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776129.467275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776129.467281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776129.467284:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776129.467286:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776129.467288:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776129.467291:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776129.467293:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776129.467295:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776129.467296:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776129.467299:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776129.467300:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776129.467300:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776129.467301:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776129.467302:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776129.467303:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776129.467304:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776129.467305:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776129.467394:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776129.467399:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776129.467402:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776130.491234:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776130.491240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776130.491246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776130.491247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776130.491250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776130.491252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776130.491311:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776130.491314:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776130.491317:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776131.515266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776131.515273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776131.515277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776131.515278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776131.515281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776131.515281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776131.515340:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776131.515345:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776131.515349:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776132.539241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776132.539248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776132.539255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776132.539256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776132.539260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776132.539261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776133.243281:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307688198, at 4307688256 00000100:00000010:0.0:1775776133.243303:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e27e0. 00010000:00000001:3.0:1775776133.563297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776133.563304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776133.563312:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776133.563313:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776133.563318:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776133.563319:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776134.459284:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776134.459294:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776134.459298:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776134.459304:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776134.459310:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776134.459318:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13017 00000400:00000200:3.0:1775776134.459329:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776134.459335:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776134.459341:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776134.459345:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776134.459348:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776134.459350:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776134.459352:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776134.459355:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776134.459356:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776134.459358:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776134.459360:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776134.459362:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776134.459363:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776134.459365:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776134.459367:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776134.459368:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776134.459369:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776134.459371:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776134.459373:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776134.459374:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776134.459378:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776134.459381:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776134.459383:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776134.459385:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776134.459387:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776134.459397:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73b80. 00000020:00000040:3.0:1775776134.459399:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776134.459403:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776134.459404:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776134.459407:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776134.459409:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776134.459413:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776134.459421:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000100:00000001:3.0:1775776134.459427:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776134.459435:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73b80 x1862022710391552/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776134.459443:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776134.459447:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776134.459458:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c52008f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73b80 x1862022710391552/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776134.459483:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776134.459487:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776134.459490:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776134.459491:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776134.459492:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776134.459495:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776134.459497:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13017 00000400:00000200:3.0:1775776134.459501:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776134.459505:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776134.459508:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776134.459511:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776134.459514:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776134.459520:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776134.459525:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776134.459530:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776134.459532:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776134.459536:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776134.459539:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776134.459543:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776134.459547:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776134.459554:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776134.459558:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776134.459563:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776134.459566:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776134.459568:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776134.459571:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776134.459573:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776134.459575:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776134.459581:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776134.459586:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776134.459591:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776134.459596:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776134.459599:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776134.459601:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776134.459607:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775776134.459611:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776134.459616:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776134.459617:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776134.459618:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776134.459620:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776134.459623:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776134.459628:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775776134.459633:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776134.459641:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710391680/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776134.459650:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776134.459654:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776134.459659:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710391680/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776134.459670:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776134.459673:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776134.459681:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776134.459684:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776134.459687:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776134.459690:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776134.459694:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13017 00000400:00000200:3.0:1775776134.459704:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776134.459710:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776134.459713:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776134.459719:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776134.459723:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776134.459727:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776134.459730:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776134.459732:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776134.459733:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776134.459735:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776134.459739:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776134.459742:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776134.459745:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776134.459752:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776134.459757:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776134.459762:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776134.459766:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776134.459771:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776134.459776:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776134.459778:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776134.459781:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776134.459785:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776134.459790:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776134.459794:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776134.459796:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776134.459798:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776134.459800:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776134.459818:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775776134.459821:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776134.459823:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776134.459824:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776134.459825:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776134.459828:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776134.459830:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776134.459834:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000100:00000001:3.0:1775776134.459839:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776134.459848:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710391808/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776134.459856:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776134.459860:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776134.459862:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710391808/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776134.459874:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776134.459877:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775776134.459880:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776134.459883:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775776134.459884:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000100:00000040:3.0:1775776134.459887:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776134.459893:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776134.459899:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13027) 00000800:00000200:1.1:1775776134.460241:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776134.460243:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775776134.460429:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776134.460432:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00010000:00000001:3.0:1775776134.587207:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776134.587215:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776134.587217:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776134.587219:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776134.587224:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776134.587226:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776134.587227:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776134.587229:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776134.587231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776134.587232:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776134.587235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776134.587236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776134.587290:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776134.587294:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776134.587297:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776135.611249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776135.611254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776135.611259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776135.611260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776135.611264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776135.611266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776135.611355:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776135.611359:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776135.611363:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776136.635273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776136.635281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776136.635289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776136.635290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776136.635294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776136.635295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776136.635354:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776136.635359:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776136.635363:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776137.659249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776137.659257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776137.659263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776137.659264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776137.659268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776137.659270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776138.363286:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307693318, at 4307693376 00000100:00000010:0.0:1775776138.363305:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2e40. 00010000:00000001:3.0:1775776138.683279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776138.683286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776138.683292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776138.683293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776138.683297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776138.683298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776139.579257:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776139.579267:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776139.579270:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776139.579274:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776139.579281:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776139.579288:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13022 00000400:00000200:3.0:1775776139.579299:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776139.579307:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776139.579309:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776139.579311:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776139.579314:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776139.579316:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776139.579319:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776139.579323:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776139.579325:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776139.579328:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776139.579332:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776139.579335:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776139.579337:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776139.579341:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776139.579343:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776139.579346:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776139.579348:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776139.579352:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776139.579356:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776139.579358:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776139.579369:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776139.579372:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776139.579374:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776139.579375:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776139.579378:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776139.579388:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775776139.579391:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776139.579395:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776139.579397:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776139.579400:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776139.579403:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776139.579408:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776139.579418:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775776139.579425:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776139.579435:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710391936/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776139.579445:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776139.579450:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776139.579453:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710391936/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776139.579471:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776139.579474:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776139.579476:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776139.579477:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776139.579478:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776139.579480:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776139.579481:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13022 00000400:00000200:3.0:1775776139.579484:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776139.579493:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776139.579495:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776139.579497:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776139.579499:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776139.579500:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776139.579502:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776139.579505:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776139.579506:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776139.579509:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776139.579510:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776139.579513:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776139.579515:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776139.579518:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776139.579519:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776139.579521:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776139.579523:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776139.579527:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776139.579531:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776139.579537:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776139.579541:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776139.579548:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776139.579551:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776139.579553:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776139.579559:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776139.579564:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776139.579568:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776139.579576:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775776139.579580:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776139.579583:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776139.579585:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776139.579587:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776139.579590:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776139.579594:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776139.579599:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000100:00000001:3.0:1775776139.579603:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776139.579612:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710392064/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776139.579621:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776139.579625:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776139.579630:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710392064/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776139.579642:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776139.579647:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776139.579650:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776139.579652:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776139.579653:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776139.579657:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776139.579661:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13022 00000400:00000200:3.0:1775776139.579667:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776139.579671:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776139.579674:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776139.579679:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776139.579681:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776139.579683:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776139.579684:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776139.579685:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776139.579686:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776139.579687:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776139.579688:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776139.579691:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776139.579693:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776139.579695:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776139.579699:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776139.579700:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776139.579701:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776139.579704:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776139.579708:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776139.579711:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776139.579712:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776139.579715:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776139.579732:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776139.579735:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776139.579739:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776139.579743:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776139.579746:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776139.579751:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000020:00000040:3.0:1775776139.579754:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776139.579760:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776139.579762:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776139.579765:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776139.579768:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776139.579772:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776139.579778:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000100:00000001:3.0:1775776139.579783:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776139.579789:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710392192/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776139.579802:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776139.579804:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776139.579809:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710392192/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776139.579815:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776139.579818:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776139.579820:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776139.579826:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776139.579833:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775776139.579836:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776139.579839:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000100:00000040:3.0:1775776139.579841:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13032) 00000800:00000200:1.1:1775776139.580151:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776139.580153:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775776139.580325:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776139.580326:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00010000:00000001:3.0:1775776139.707245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776139.707252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776139.707259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776139.707263:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776139.707266:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776139.707270:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776139.707274:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776139.707277:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776139.707279:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776139.707281:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776139.707284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776139.707286:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776139.707287:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776139.707288:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776139.707289:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776139.707289:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776139.707291:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776139.707292:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776139.707369:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776139.707374:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776139.707378:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776140.731269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776140.731276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776140.731283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776140.731284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776140.731288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776140.731289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776140.731344:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776140.731349:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776140.731353:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776141.755276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776141.755283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776141.755290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776141.755292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776141.755296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776141.755297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776141.755349:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776141.755353:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776141.755355:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776142.779286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776142.779293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776142.779300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776142.779301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776142.779305:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776142.779306:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776143.483223:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307698438, at 4307698496 00000800:00000200:2.0:1775776143.483234:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000100:00000010:0.0:1775776143.483237:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e28a0. 00000800:00000200:2.0:1775776143.483240:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775776143.483244:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1775776143.483363:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776143.483377:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775776143.483381:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776143.483383:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1775776143.803218:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776143.803223:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776143.803228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776143.803229:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776143.803233:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776143.803234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776144.699252:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776144.699258:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776144.699262:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776144.699264:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776144.699269:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776144.699276:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13027 00000400:00000200:2.0:1775776144.699284:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776144.699289:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776144.699291:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776144.699293:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776144.699295:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776144.699298:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776144.699299:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776144.699302:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776144.699303:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776144.699305:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776144.699307:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776144.699309:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776144.699310:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776144.699312:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776144.699314:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776144.699315:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776144.699316:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776144.699318:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776144.699320:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776144.699322:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776144.699324:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776144.699326:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776144.699328:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776144.699329:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776144.699330:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776144.699337:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775776144.699339:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776144.699342:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776144.699342:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776144.699344:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776144.699346:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776144.699349:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776144.699354:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775776144.699358:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776144.699364:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710392320/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776144.699372:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776144.699375:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776144.699378:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710392320/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776144.699403:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776144.699405:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776144.699407:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776144.699408:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776144.699409:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776144.699411:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776144.699413:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13027 00000400:00000200:2.0:1775776144.699415:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776144.699418:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776144.699420:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776144.699422:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776144.699424:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776144.699425:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776144.699427:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776144.699429:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776144.699430:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776144.699432:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776144.699434:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776144.699435:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776144.699438:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776144.699441:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776144.699442:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776144.699445:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776144.699447:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776144.699449:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776144.699453:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776144.699456:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776144.699459:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776144.699464:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776144.699468:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776144.699472:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776144.699475:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776144.699478:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776144.699482:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776144.699484:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775776144.699487:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776144.699490:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776144.699493:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776144.699496:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776144.699498:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776144.699501:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776144.699504:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775776144.699508:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776144.699515:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710392448/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776144.699521:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776144.699524:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776144.699528:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710392448/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776144.699536:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776144.699540:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776144.699542:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776144.699543:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776144.699545:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776144.699548:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776144.699550:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13027 00000400:00000200:2.0:1775776144.699555:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776144.699557:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776144.699559:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776144.699561:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776144.699564:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776144.699568:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776144.699572:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776144.699574:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776144.699576:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776144.699579:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776144.699582:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776144.699586:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776144.699591:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776144.699598:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776144.699601:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776144.699607:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776144.699610:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776144.699615:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776144.699619:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776144.699623:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776144.699626:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776144.699642:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776144.699645:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776144.699647:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776144.699650:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776144.699654:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776144.699656:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776144.699662:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775776144.699666:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776144.699670:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776144.699672:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776144.699673:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776144.699676:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776144.699678:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776144.699682:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775776144.699686:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776144.699694:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710392576/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776144.699704:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776144.699706:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776144.699711:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710392576/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776144.699725:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776144.699730:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776144.699733:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776144.699737:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776144.699739:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776144.699744:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13037) 00000800:00000200:1.1:1775776144.699751:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776144.699753:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775776144.700008:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776144.700010:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775776144.700311:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776144.700314:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00010000:00000001:3.0:1775776144.827269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776144.827279:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776144.827282:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776144.827286:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776144.827291:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776144.827293:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776144.827296:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776144.827298:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776144.827302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776144.827303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776144.827305:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776144.827306:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776144.827396:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776144.827402:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776144.827406:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776145.851300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776145.851310:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776145.851318:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776145.851320:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776145.851323:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776145.851325:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776145.851396:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776145.851401:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776145.851405:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776146.875265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776146.875270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776146.875274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776146.875275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776146.875278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776146.875279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776146.875330:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776146.875334:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776146.875338:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776147.899288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776147.899296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776147.899303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776147.899304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776147.899309:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776147.899310:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776148.603262:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307703558, at 4307703616 00000100:00000010:0.0:1775776148.603281:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2a20. 00010000:00000001:3.0:1775776148.923284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776148.923290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776148.923297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776148.923299:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776148.923302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776148.923303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776149.819250:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776149.819257:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776149.819267:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776149.819273:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776149.819280:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776149.819291:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13032 00000400:00000200:3.0:1775776149.819302:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776149.819310:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776149.819314:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776149.819317:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776149.819321:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776149.819323:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776149.819326:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776149.819329:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776149.819331:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776149.819333:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776149.819336:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776149.819338:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776149.819339:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776149.819341:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776149.819342:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776149.819344:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776149.819345:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776149.819346:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776149.819349:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776149.819350:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776149.819353:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776149.819355:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776149.819357:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776149.819359:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776149.819362:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776149.819371:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775776149.819374:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776149.819377:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776149.819378:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776149.819381:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776149.819383:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776149.819387:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776149.819396:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000100:00000001:3.0:1775776149.819402:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776149.819410:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710392704/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776149.819417:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776149.819421:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776149.819425:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710392704/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776149.819441:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776149.819443:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776149.819445:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776149.819446:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776149.819447:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776149.819449:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776149.819451:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13032 00000400:00000200:3.0:1775776149.819458:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776149.819461:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776149.819465:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776149.819469:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776149.819471:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776149.819475:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776149.819478:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776149.819484:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776149.819488:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776149.819492:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776149.819495:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776149.819499:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776149.819505:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776149.819511:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776149.819513:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776149.819518:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776149.819522:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776149.819526:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776149.819531:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776149.819536:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776149.819562:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776149.819571:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776149.819580:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776149.819582:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776149.819589:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776149.819592:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776149.819597:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776149.819603:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775776149.819607:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776149.819612:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776149.819614:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776149.819618:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776149.819620:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776149.819623:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776149.819628:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000001:3.0:1775776149.819632:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776149.819639:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710392832/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776149.819651:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776149.819655:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776149.819663:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710392832/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776149.819678:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776149.819683:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776149.819689:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776149.819692:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776149.819695:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776149.819700:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776149.819705:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13032 00000400:00000200:3.0:1775776149.819711:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776149.819714:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776149.819716:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776149.819719:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776149.819723:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776149.819728:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776149.819731:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776149.819734:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776149.819736:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776149.819737:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776149.819739:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776149.819741:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776149.819745:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776149.819749:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776149.819750:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776149.819753:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776149.819755:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776149.819756:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776149.819760:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776149.819764:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776149.819767:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776149.819771:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776149.819775:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776149.819775:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775776149.819778:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:1.1:1775776149.819780:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000008:00100000:3.0:1775776149.819781:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776149.819786:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776149.819790:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776149.819795:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775776149.819798:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776149.819800:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776149.819802:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776149.819804:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776149.819807:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776149.819810:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776149.819814:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000001:3.0:1775776149.819817:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776149.819822:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710392960/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776149.819829:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776149.819831:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776149.819836:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710392960/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776149.819858:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776149.819860:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776149.819863:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776149.819866:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776149.819869:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776149.819875:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13042) 00000800:00000200:1.1:1775776149.820159:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776149.820161:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775776149.820407:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776149.820409:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00010000:00000001:3.0:1775776149.947219:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776149.947224:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776149.947230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776149.947233:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776149.947235:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776149.947238:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776149.947240:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776149.947242:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776149.947244:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776149.947246:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776149.947249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776149.947249:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776149.947250:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776149.947251:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776149.947252:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776149.947252:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776149.947254:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776149.947255:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776149.947308:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776149.947312:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776149.947316:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776150.971273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776150.971281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776150.971289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776150.971290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776150.971294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776150.971295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776150.971347:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776150.971352:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776150.971355:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776151.995244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776151.995250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776151.995254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776151.995255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776151.995258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776151.995258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776151.995312:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776151.995316:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776151.995319:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776153.019272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776153.019278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776153.019284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776153.019285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776153.019288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776153.019289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776153.723258:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307708678, at 4307708736 00000100:00000010:0.0:1775776153.723274:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e28a0. 00010000:00000001:3.0:1775776154.043267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776154.043274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776154.043279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776154.043280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776154.043283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776154.043284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776154.939266:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776154.939275:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776154.939278:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776154.939282:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776154.939286:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776154.939293:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13037 00000400:00000200:3.0:1775776154.939302:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776154.939307:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776154.939310:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776154.939312:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776154.939315:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776154.939317:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776154.939318:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776154.939322:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776154.939323:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776154.939326:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776154.939328:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776154.939330:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776154.939331:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776154.939333:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776154.939334:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776154.939336:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776154.939337:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776154.939339:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776154.939341:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776154.939342:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776154.939345:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776154.939347:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776154.939349:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776154.939352:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776154.939353:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776154.939363:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775776154.939365:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776154.939369:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776154.939370:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776154.939373:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776154.939375:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776154.939379:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776154.939389:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775776154.939396:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776154.939407:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710393088/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776154.939418:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776154.939423:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776154.939428:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710393088/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776154.939465:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776154.939469:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776154.939473:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776154.939474:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776154.939476:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776154.939479:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776154.939481:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13037 00000400:00000200:3.0:1775776154.939485:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776154.939488:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776154.939491:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776154.939493:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776154.939496:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776154.939498:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776154.939500:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776154.939503:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776154.939504:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776154.939506:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776154.939508:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776154.939510:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776154.939512:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776154.939515:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776154.939517:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776154.939519:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776154.939520:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776154.939522:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776154.939524:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776154.939525:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776154.939527:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776154.939531:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776154.939532:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776154.939533:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776154.939537:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776154.939539:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776154.939540:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776154.939543:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000020:00000040:3.0:1775776154.939545:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776154.939548:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776154.939549:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776154.939551:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776154.939553:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776154.939554:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776154.939559:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:3.0:1775776154.939562:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776154.939567:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710393216/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776154.939574:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776154.939577:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776154.939580:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710393216/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776154.939588:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776154.939590:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776154.939593:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776154.939594:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776154.939596:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776154.939598:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776154.939601:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13037 00000400:00000200:3.0:1775776154.939604:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776154.939606:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776154.939608:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776154.939610:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776154.939612:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776154.939613:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776154.939615:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776154.939617:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776154.939618:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776154.939620:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776154.939621:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776154.939623:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776154.939625:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776154.939627:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776154.939629:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776154.939630:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776154.939632:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776154.939634:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776154.939636:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776154.939639:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776154.939641:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776154.939644:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776154.939646:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776154.939648:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776154.939650:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776154.939652:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776154.939654:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776154.939657:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775776154.939658:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776154.939660:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776154.939661:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776154.939662:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776154.939663:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776154.939665:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776154.939667:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000001:3.0:1775776154.939669:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776154.939672:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710393344/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776154.939677:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776154.939678:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776154.939681:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710393344/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776154.939686:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776154.939687:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776154.939690:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776154.939693:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776154.939695:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776154.939699:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13047) 00000100:00000001:2.0:1775776154.939826:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776154.939831:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776154.939835:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776154.939841:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776154.939844:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776154.939847:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776154.939850:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776154.939852:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776154.939854:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776154.939962:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776154.939964:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775776154.940226:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776154.940227:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775776154.940391:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776154.940393:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00010000:00000001:3.0:1775776155.067254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776155.067264:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776155.067266:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776155.067270:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776155.067275:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776155.067278:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776155.067280:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776155.067283:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776155.067287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776155.067288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776155.067292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776155.067293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776155.067359:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776155.067364:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776155.067368:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776155.963198:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776155.963206:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776155.963208:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776155.963214:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776155.963218:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776155.963222:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776155.963225:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776155.963227:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776155.963230:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776156.091287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776156.091294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776156.091299:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776156.091300:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776156.091304:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776156.091305:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776156.091366:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776156.091370:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776156.091373:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776156.987287:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776156.987294:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776156.987295:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776156.987300:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776156.987303:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776156.987307:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776156.987310:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776156.987312:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776156.987314:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776157.115199:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776157.115205:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776157.115211:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776157.115212:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776157.115216:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776157.115217:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776157.115301:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776157.115306:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776157.115309:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1775776158.139278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776158.139285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776158.139292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776158.139293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776158.139297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776158.139298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776158.843255:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307713798, at 4307713856 00000100:00000010:0.0:1775776158.843274:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2e40. 00010000:00000001:3.0:1775776159.163214:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776159.163220:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776159.163224:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776159.163225:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776159.163228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776159.163228:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776160.059255:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776160.059263:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776160.059266:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776160.059269:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776160.059275:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776160.059284:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13042 00000400:00000200:3.0:1775776160.059294:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776160.059301:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776160.059308:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776160.059314:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776160.059318:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776160.059321:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776160.059323:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776160.059326:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776160.059328:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776160.059330:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776160.059332:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776160.059333:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776160.059335:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776160.059337:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776160.059339:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776160.059340:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776160.059342:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776160.059343:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776160.059346:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776160.059348:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776160.059350:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776160.059353:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776160.059355:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776160.059358:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776160.059382:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776160.059393:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73100. 00000020:00000040:3.0:1775776160.059397:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776160.059401:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776160.059403:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776160.059406:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776160.059408:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776160.059412:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776160.059418:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:3.0:1775776160.059423:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776160.059429:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73100 x1862022710393472/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776160.059436:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776160.059439:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776160.059442:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f67e6552] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73100 x1862022710393472/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776160.059459:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776160.059461:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776160.059463:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776160.059464:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776160.059465:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776160.059468:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776160.059470:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13042 00000400:00000200:3.0:1775776160.059473:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776160.059477:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776160.059479:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776160.059481:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776160.059483:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776160.059484:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776160.059486:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776160.059489:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776160.059491:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776160.059493:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776160.059497:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776160.059501:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776160.059503:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776160.059506:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776160.059507:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776160.059511:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776160.059512:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776160.059514:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776160.059519:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776160.059525:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776160.059531:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776160.059538:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776160.059543:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776160.059547:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776160.059552:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776160.059557:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776160.059561:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776160.059569:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775776160.059574:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776160.059583:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776160.059586:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776160.059589:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776160.059590:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776160.059592:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776160.059599:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000001:3.0:1775776160.059604:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776160.059612:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710393600/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776160.059627:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776160.059632:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776160.059641:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710393600/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776160.059681:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776160.059689:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776160.059701:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776160.059706:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776160.059710:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776160.059714:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776160.059719:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13042 00000400:00000200:3.0:1775776160.059727:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776160.059732:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776160.059735:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776160.059744:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776160.059746:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776160.059747:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776160.059749:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776160.059750:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776160.059751:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776160.059753:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776160.059756:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776160.059760:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776160.059762:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776160.059766:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776160.059768:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776160.059776:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776160.059784:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776160.059787:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776160.059792:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776160.059794:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776160.059796:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776160.059798:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776160.059802:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776160.059804:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776160.059806:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776160.059810:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776160.059815:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776160.059822:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000020:00000040:3.0:1775776160.059825:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776160.059830:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776160.059832:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776160.059837:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776160.059839:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776160.059841:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776160.059845:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000100:00000001:3.0:1775776160.059850:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776160.059855:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710393728/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776160.059874:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776160.059878:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776160.059886:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710393728/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776160.059893:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776160.059898:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776160.059907:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776160.059908:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776160.059911:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776160.059918:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13052) 00000800:00000200:1.1:1775776160.059935:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776160.059939:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775776160.059943:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776160.059948:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776160.059956:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776160.059962:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776160.059968:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3c38 00000400:00000010:1.0:1775776160.059971:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3c38. 00000100:00000001:1.0:1775776160.059978:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776160.059982:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73100 x1862022710393472/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.059993:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776160.059996:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73100 x1862022710393472/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776185 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060003:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776160.060005:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776160.060012:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776160.060016:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776160.060019:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307718918 00000100:00000010:1.0:1775776160.060024:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e27e0. 00000100:00000001:1.0:1775776160.060032:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776160.060050:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307718918 00000100:00000010:1.0:1775776160.060054:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2ba0. 00000100:00000001:1.0:1775776160.060061:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776160.060065:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776160.060068:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776160.060070:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776160.060074:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776160/real 1775776160] req@ffff8f6202c73100 x1862022710393472/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776160.060085:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776160.060093:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776160.060096:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776160.060099:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776160.060100:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776160.060104:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776160.060108:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73100 x1862022710393472/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776160.060117:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73100 x1862022710393472/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060123:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776160.060125:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776160.060127:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776160.060129:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776160.060131:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73100 x1862022710393472/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060156:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776160.060162:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73100 x1862022710393472/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060169:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776160.060172:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776160.060173:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776160.060175:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776160.060177:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776160.060179:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776160.060183:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776160.060187:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776160.060189:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776160.060193:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73100 x1862022710393472/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776160.060202:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710393472:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776160.060207:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776160.060209:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776160.060212:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73100 x1862022710393472/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060224:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776160.060226:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776160.060229:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776160.060232:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776160.060234:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776160.060236:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776160.060238:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776160.060241:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37c00. 02000000:00000001:1.0:1775776160.060245:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776160.060247:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776160.060249:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73100. 00000100:00000001:1.0:1775776160.060253:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776160.060254:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776160.060256:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776160.060263:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776160.060265:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776160.060267:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776160.060268:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776160.060270:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776160.060273:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70000 x1862022710393600/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060280:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776160.060281:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776160.060283:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776160.060285:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776160.060287:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776160.060289:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710393600:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776160.060292:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776160.060294:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776160.060296:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776160.060297:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776160.060300:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776160.060301:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776160.060305:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776160.060308:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776160.060312:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776160.060315:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710393600, portal 10 00000100:00000001:1.0:1775776160.060317:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776160.060318:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004032 : -123823860547584 : ffff8f6202c70000) 00000100:00000040:1.0:1775776160.060325:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70000 x1862022710393600/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060332:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776160.060334:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776160.060336:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3c38. 00000100:00000200:1.0:1775776160.060339:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710393600, offset 0 00000400:00000200:1.0:1775776160.060342:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776160.060346:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776160.060350:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776160.060353:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776160.060355:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776160.060357:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776160.060373:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776160.060376:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453627:453627:256:4294967295] 192.168.201.115@tcp LPNI seq info [453552:453552:8:4294967295] 00000400:00000200:1.0:1775776160.060381:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776160.060387:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776160.060391:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776160.060395:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776160.060398:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776160.060401:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775776160.060405:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776160.060407:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775776160.060411:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776160.060412:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776160.060415:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776160.060419:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776160.060422:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3c38 00000400:00000010:1.0:1775776160.060423:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3c38. 00000100:00000001:1.0:1775776160.060427:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776160.060430:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70000 x1862022710393600/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060438:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776160.060440:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70000 x1862022710393600/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776185 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060445:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776160.060446:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776160.060448:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776160.060451:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776160.060454:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776160.060455:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307715084 00000100:00000001:1.0:1775776160.060457:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307718918 00000100:00000001:1.0:1775776160.060459:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307715084 00000100:00000001:1.0:1775776160.060460:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307718918 00000100:00000001:1.0:1775776160.060461:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776160.060463:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776160.060464:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776160.060466:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776160/real 1775776160] req@ffff8f6202c70000 x1862022710393600/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776160.060470:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776160.060471:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776160.060472:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776160.060473:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776160.060474:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776160.060475:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776160.060476:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70000 x1862022710393600/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776160.060481:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70000 x1862022710393600/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060489:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776160.060490:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776160.060491:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776160.060492:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776160.060494:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70000 x1862022710393600/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060499:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776160.060502:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70000 x1862022710393600/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060508:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776160.060510:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776160.060512:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776160.060513:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776160.060515:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776160.060520:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776160.060523:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776160.060526:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776160.060528:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776160.060532:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70000 x1862022710393600/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776160.060540:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710393600:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776160.060544:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776160.060547:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70e00 x1862022710393728/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060554:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776160.060555:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776160.060558:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776160.060559:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776160.060561:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776160.060564:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710393728:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776160.060567:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776160.060569:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776160.060570:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776160.060572:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776160.060575:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776160.060577:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776160.060580:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776160.060583:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776160.060586:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776160.060588:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710393728, portal 10 00000100:00000001:1.0:1775776160.060590:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776160.060590:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849007616 : -123823860544000 : ffff8f6202c70e00) 00000100:00000040:1.0:1775776160.060595:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70e00 x1862022710393728/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060599:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776160.060600:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776160.060601:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3c38. 00000100:00000200:1.0:1775776160.060603:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710393728, offset 0 00000400:00000200:1.0:1775776160.060605:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776160.060607:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776160.060609:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776160.060611:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776160.060613:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776160.060614:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776160.060616:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776160.060618:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453628:453628:256:4294967295] 192.168.201.115@tcp LPNI seq info [453553:453553:8:4294967295] 00000400:00000200:1.0:1775776160.060620:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776160.060624:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776160.060628:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776160.060630:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776160.060632:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776160.060634:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775776160.060636:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776160.060638:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775776160.060640:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776160.060641:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776160.060643:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776160.060645:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776160.060647:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3c38 00000400:00000010:1.0:1775776160.060648:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3c38. 00000100:00000001:1.0:1775776160.060650:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776160.060652:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70e00 x1862022710393728/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060657:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776160.060658:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70e00 x1862022710393728/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776185 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060662:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776160.060663:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776160.060664:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776160.060666:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776160.060668:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776160.060669:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307715084 00000100:00000001:1.0:1775776160.060670:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307718918 00000100:00000001:1.0:1775776160.060671:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307715084 00000100:00000001:1.0:1775776160.060673:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307718918 00000100:00000001:1.0:1775776160.060674:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776160.060675:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776160.060676:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776160.060678:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776160/real 1775776160] req@ffff8f6202c70e00 x1862022710393728/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776160.060683:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776160.060684:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776160.060685:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776160.060686:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776160.060687:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776160.060688:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776160.060689:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70e00 x1862022710393728/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776160.060693:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70e00 x1862022710393728/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060696:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776160.060697:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776160.060698:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776160.060699:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776160.060701:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70e00 x1862022710393728/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060704:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776160.060706:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70e00 x1862022710393728/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060710:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776160.060711:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776160.060712:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776160.060713:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776160.060714:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776160.060715:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776160.060717:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776160.060719:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776160.060720:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776160.060723:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70e00 x1862022710393728/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776160.060727:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710393728:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776160.060730:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776160.060730:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776160.060732:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70000 x1862022710393600/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060735:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776160.060736:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776160.060737:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776160.060738:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776160.060739:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776160.060740:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776160.060741:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776160.060742:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37800. 02000000:00000001:1.0:1775776160.060743:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776160.060744:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776160.060744:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70000. 00000100:00000001:1.0:1775776160.060746:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776160.060747:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776160.060748:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776160.060750:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70e00 x1862022710393728/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776185 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776160.060753:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776160.060754:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776160.060754:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776160.060755:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776160.060756:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776160.060757:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776160.060758:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776160.060758:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35000. 02000000:00000001:1.0:1775776160.060759:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776160.060760:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776160.060761:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70e00. 00000100:00000001:1.0:1775776160.060763:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776160.060764:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776160.060765:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776160.060767:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776160.060768:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776160.060769:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776160.060770:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776160.187261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776160.187266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776160.187272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776160.187275:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776160.187277:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776160.187280:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776160.187283:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776160.187285:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776160.187286:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776160.187289:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776160.187293:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776160.187294:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776160.187296:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776160.187297:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776160.187298:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776160.187300:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776160.187301:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776160.187303:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776160.187395:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776160.187398:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776160.187400:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776161.083254:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776161.083261:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776161.083264:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776161.083268:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776161.083272:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776161.083276:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776161.083278:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776161.083280:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776161.083283:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776161.211188:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776161.211192:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776161.211195:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776161.211196:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776161.211198:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776161.211199:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776161.211254:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776161.211258:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776161.211260:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776162.107227:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776162.107234:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776162.107235:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776162.107240:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776162.107243:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776162.107246:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776162.107249:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776162.107251:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776162.107253:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776162.235306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776162.235315:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776162.235350:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776162.235352:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776162.235356:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776162.235357:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776162.235412:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776162.235416:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776162.235420:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776163.131252:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776163.131261:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776163.131263:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776163.131286:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776163.131291:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776163.131295:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776163.131299:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776163.131301:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776163.131305:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776163.259343:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776163.259351:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776163.259358:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776163.259359:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776163.259363:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776163.259364:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776163.259426:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776163.259431:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776163.259434:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776163.963240:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307718918, at 4307718976 00000100:00000040:1.0:1775776163.963240:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307718918, at 4307718976 00000100:00000010:1.0:1775776163.963255:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ba0. 00000100:00000010:0.0:1775776163.963257:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e27e0. 00000100:00000001:1.0:1775776164.155297:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776164.155304:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776164.155306:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776164.155312:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776164.155315:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776164.155319:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776164.155321:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776164.155324:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776164.155326:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776164.283216:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776164.283223:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776164.283228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776164.283229:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776164.283235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776164.283236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776164.283300:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776164.283303:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776164.283306:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776165.179266:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776165.179288:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776165.179294:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776165.179297:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776165.179304:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:3.0:1775776165.179304:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776165.179308:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776165.179309:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776165.179313:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776165.179315:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776165.179319:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776165.179320:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776165.179324:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776165.179327:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776165.179329:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13047 00000400:00000200:3.0:1775776165.179341:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776165.179348:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776165.179352:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776165.179356:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776165.179360:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776165.179363:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776165.179365:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776165.179376:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776165.179381:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776165.179385:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776165.179388:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776165.179390:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776165.179393:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776165.179396:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776165.179398:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776165.179400:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776165.179402:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776165.179405:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776165.179408:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776165.179410:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776165.179414:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776165.179417:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776165.179421:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776165.179424:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776165.179427:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776165.179437:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775776165.179441:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776165.179445:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776165.179447:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776165.179450:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776165.179453:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776165.179458:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776165.179468:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000100:00000001:3.0:1775776165.179476:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776165.179487:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710393856/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776165.179497:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776165.179503:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776165.179508:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710393856/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776165.179531:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776165.179534:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776165.179537:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776165.179539:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776165.179540:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776165.179543:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776165.179545:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13047 00000400:00000200:3.0:1775776165.179550:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776165.179554:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776165.179556:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776165.179559:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776165.179560:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776165.179562:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776165.179565:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775776165.179565:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776165.179569:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776165.179569:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776165.179574:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776165.179575:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776165.179578:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000040:1.0:1775776165.179579:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71c00 x1862022710393856/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775776165.179580:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776165.179584:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776165.179589:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776165.179598:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1775776165.179598:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776165.179601:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776165.179602:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776165.179605:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776165.179606:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1775776165.179609:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776165.179611:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1775776165.179613:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:1.0:1775776165.179616:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776165.179618:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776165.179623:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776165.179624:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710393856:192.168.201.115@tcp:250:kworker.0 02000000:00000001:3.0:1775776165.179627:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776165.179633:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776165.179635:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776165.179638:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776165.179638:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776165.179642:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776165.179642:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776165.179648:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:3.0:1775776165.179649:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776165.179653:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776165.179654:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d000. 00000100:00000001:3.0:1775776165.179656:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776165.179658:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776165.179662:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000400:00000200:1.0:1775776165.179667:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000040:3.0:1775776165.179667:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776165.179672:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776165.179673:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776165.179674:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000100:00000001:3.0:1775776165.179676:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776165.179679:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776165.179681:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776165.179684:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 02000000:00000010:3.0:1775776165.179686:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000100:00000001:3.0:1775776165.179690:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776165.179694:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710393856, portal 25 00000100:00100000:3.0:1775776165.179697:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710393984/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776165.179698:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776165.179701:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849011200 : -123823860540416 : ffff8f6202c71c00) 00000100:00000001:3.0:1775776165.179711:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1775776165.179712:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71c00 x1862022710393856/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776165.179716:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776165.179724:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710393984/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.179726:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776165.179734:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776165.179739:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3c38. 00000100:00000001:3.0:1775776165.179741:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776165.179743:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710393856, offset 0 00000100:00080000:3.0:1775776165.179745:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776165.179750:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776165.179754:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1775776165.179755:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00080000:3.0:1775776165.179756:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776165.179760:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776165.179764:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13047 00000400:00000200:1.0:1775776165.179768:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776165.179772:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776165.179777:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000400:00000200:1.0:1775776165.179779:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00080000:3.0:1775776165.179781:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776165.179786:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776165.179788:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1775776165.179791:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776165.179795:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776165.179797:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:3.0:1775776165.179798:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776165.179802:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:1.0:1775776165.179804:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000020:00000040:3.0:1775776165.179805:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776165.179809:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:1.0:1775776165.179811:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000020:00000040:3.0:1775776165.179813:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776165.179816:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776165.179819:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453629:453629:256:4294967295] 192.168.201.115@tcp LPNI seq info [453554:453554:8:4294967295] 00000020:00000001:3.0:1775776165.179822:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776165.179829:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775776165.179830:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776165.179834:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776165.179836:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776165.179839:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776165.179843:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776165.179850:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00080000:3.0:1775776165.179850:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776165.179856:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776165.179859:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:1.0:1775776165.179863:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 02000000:00000001:3.0:1775776165.179864:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776165.179868:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775776165.179870:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000100:00000001:3.0:1775776165.179870:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776165.179873:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776165.179877:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776165.179881:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775776165.179881:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776165.179888:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000100:00000010:3.0:1775776165.179888:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775776165.179893:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776165.179899:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000800:00000200:1.1:1775776165.179899:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:3.0:1775776165.179901:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776165.179905:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.1:1775776165.179905:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 02000000:00000001:3.0:1775776165.179908:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776165.179911:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1775776165.179915:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 02000000:00000010:3.0:1775776165.179917:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000800:00000010:1.0:1775776165.179919:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000100:00000001:3.0:1775776165.179922:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776165.179928:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00100000:3.0:1775776165.179929:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710394112/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776165.179939:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1775776165.179945:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775776165.179947:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3c38 00000100:00000001:3.0:1775776165.179948:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776165.179952:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3c38. 00000100:00000040:3.0:1775776165.179956:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710394112/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.179961:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1775776165.179966:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776165.179968:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71c00 x1862022710393856/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776165.179970:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776165.179980:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776165.179986:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776165.179989:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776165.179991:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:1.0:1775776165.179995:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71c00 x1862022710393856/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776190 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776165.179999:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13057) 00000100:00000001:1.0:1775776165.180006:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776165.180008:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776165.180014:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776165.180018:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776165.180021:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307724038 00000100:00000010:1.0:1775776165.180026:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2ba0. 00000100:00000001:1.0:1775776165.180034:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776165.180041:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307724038 00000100:00000010:1.0:1775776165.180044:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e20c0. 00000100:00000001:1.0:1775776165.180047:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776165.180051:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776165.180054:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776165.180056:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776165.180060:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776165/real 1775776165] req@ffff8f6202c71c00 x1862022710393856/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776165.180069:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776165.180071:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776165.180073:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776165.180076:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776165.180077:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776165.180079:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776165.180083:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71c00 x1862022710393856/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776165.180091:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71c00 x1862022710393856/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.180097:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776165.180099:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776165.180101:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776165.180103:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776165.180107:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71c00 x1862022710393856/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.180113:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776165.180117:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71c00 x1862022710393856/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.180123:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776165.180126:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776165.180127:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776165.180129:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776165.180131:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776165.180152:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776165.180155:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776165.180159:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776165.180161:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776165.180165:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71c00 x1862022710393856/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776165.180173:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710393856:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776165.180177:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776165.180179:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776165.180181:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71c00 x1862022710393856/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.180192:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776165.180195:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776165.180198:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776165.180201:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776165.180203:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776165.180204:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776165.180206:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776165.180209:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36c00. 02000000:00000001:1.0:1775776165.180212:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776165.180214:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776165.180216:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71c00. 00000100:00000001:1.0:1775776165.180219:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776165.180220:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776165.180222:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776165.180230:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776165.180231:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776165.180233:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776165.180234:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776165.180236:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776165.180239:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73480 x1862022710393984/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.180244:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776165.180246:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776165.180247:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776165.180249:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776165.180251:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776165.180254:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710393984:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776165.180257:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776165.180258:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776165.180260:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776165.180262:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776165.180264:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776165.180266:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776165.180282:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776165.180286:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776165.180290:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776165.180293:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710393984, portal 10 00000100:00000001:1.0:1775776165.180294:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776165.180296:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849017472 : -123823860534144 : ffff8f6202c73480) 00000100:00000040:1.0:1775776165.180302:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73480 x1862022710393984/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.180308:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776165.180311:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776165.180312:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3c38. 00000100:00000200:1.0:1775776165.180315:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710393984, offset 0 00000400:00000200:1.0:1775776165.180318:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776165.180322:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776165.180325:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776165.180328:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776165.180330:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776165.180333:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776165.180335:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776165.180337:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453630:453630:256:4294967295] 192.168.201.115@tcp LPNI seq info [453555:453555:8:4294967295] 00000400:00000200:1.0:1775776165.180341:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776165.180347:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776165.180350:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776165.180353:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776165.180357:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776165.180359:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775776165.180363:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776165.180366:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1775776165.180369:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776165.180371:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776165.180374:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776165.180377:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776165.180379:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3c38 00000400:00000010:1.0:1775776165.180381:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3c38. 00000100:00000001:1.0:1775776165.180384:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776165.180386:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73480 x1862022710393984/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.180393:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776165.180396:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73480 x1862022710393984/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776190 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.180401:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776165.180403:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776165.180405:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776165.180409:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776165.180412:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776165.180414:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307720204 00000100:00000001:1.0:1775776165.180416:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307724038 00000100:00000001:1.0:1775776165.180418:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307720204 00000100:00000001:1.0:1775776165.180420:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307724038 00000100:00000001:1.0:1775776165.180422:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776165.180424:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776165.180425:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776165.180428:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776165/real 1775776165] req@ffff8f6202c73480 x1862022710393984/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776165.180435:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776165.180436:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776165.180438:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776165.180440:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776165.180441:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776165.180443:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776165.180446:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73480 x1862022710393984/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776165.180453:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73480 x1862022710393984/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.180463:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776165.180465:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776165.180466:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776165.180467:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776165.180470:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73480 x1862022710393984/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.180476:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776165.180480:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73480 x1862022710393984/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.180485:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776165.180488:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776165.180489:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776165.180490:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776165.180492:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776165.180497:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776165.180499:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776165.180502:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776165.180504:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776165.180507:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73480 x1862022710393984/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776165.180514:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710393984:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776165.180517:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776165.180520:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70380 x1862022710394112/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.180525:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776165.180527:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776165.180529:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776165.180530:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776165.180532:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776165.180534:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710394112:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776165.180537:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776165.180539:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776165.180540:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776165.180542:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776165.180545:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776165.180547:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776165.180549:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776165.180552:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776165.180554:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776165.180557:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710394112, portal 10 00000100:00000001:1.0:1775776165.180558:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776165.180560:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004928 : -123823860546688 : ffff8f6202c70380) 00000100:00000040:1.0:1775776165.180564:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70380 x1862022710394112/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.180570:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776165.180572:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776165.180574:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3c38. 00000100:00000200:1.0:1775776165.180576:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710394112, offset 0 00000400:00000200:1.0:1775776165.180579:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776165.180582:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776165.180585:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776165.180588:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776165.180590:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776165.180592:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776165.180594:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776165.180597:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453631:453631:256:4294967295] 192.168.201.115@tcp LPNI seq info [453556:453556:8:4294967295] 00000400:00000200:1.0:1775776165.180601:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776165.180606:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776165.180610:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776165.180613:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776165.180616:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776165.180619:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775776165.180622:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776165.180624:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1775776165.180627:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776165.180629:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776165.180631:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776165.180634:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776165.180637:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3c38 00000400:00000010:1.0:1775776165.180638:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3c38. 00000100:00000001:1.0:1775776165.180640:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776165.180643:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70380 x1862022710394112/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.180649:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776165.180652:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70380 x1862022710394112/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776190 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.180657:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776165.180659:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776165.180661:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776165.180664:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776165.180666:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776165.180668:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307720204 00000100:00000001:1.0:1775776165.180670:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307724038 00000100:00000001:1.0:1775776165.180672:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307720204 00000100:00000001:1.0:1775776165.180674:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307724038 00000100:00000001:1.0:1775776165.180676:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776165.180678:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776165.180680:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776165.180682:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776165/real 1775776165] req@ffff8f6202c70380 x1862022710394112/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776165.180690:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776165.180691:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776165.180692:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776165.180694:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776165.180695:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776165.180697:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776165.180700:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70380 x1862022710394112/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776165.180707:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70380 x1862022710394112/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.180712:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776165.180713:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776165.180715:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776165.180716:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776165.180719:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70380 x1862022710394112/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.180725:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776165.180728:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70380 x1862022710394112/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.180734:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776165.180736:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776165.180738:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776165.180739:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776165.180740:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776165.180742:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776165.180744:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776165.180747:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776165.180749:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776165.180753:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70380 x1862022710394112/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776165.180759:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710394112:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776165.180763:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776165.180764:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776165.180766:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73480 x1862022710393984/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.180771:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776165.180773:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776165.180774:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776165.180776:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776165.180777:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776165.180779:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776165.180780:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776165.180782:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34000. 02000000:00000001:1.0:1775776165.180784:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776165.180785:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776165.180786:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73480. 00000100:00000001:1.0:1775776165.180788:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776165.180789:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776165.180791:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776165.180793:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70380 x1862022710394112/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776190 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776165.180798:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776165.180799:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776165.180801:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776165.180803:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776165.180804:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776165.180805:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776165.180806:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776165.180807:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35c00. 02000000:00000001:1.0:1775776165.180809:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776165.180811:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776165.180812:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70380. 00000100:00000001:1.0:1775776165.180814:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776165.180815:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776165.180817:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776165.180819:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776165.180821:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776165.180822:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776165.180824:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776165.307300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776165.307311:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776165.307314:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776165.307318:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776165.307324:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776165.307327:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776165.307329:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776165.307332:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776165.307337:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776165.307339:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776165.307342:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776165.307344:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776165.307395:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776165.307399:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776165.307402:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776166.203201:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776166.203206:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776166.203207:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776166.203211:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776166.203214:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776166.203217:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776166.203219:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776166.203220:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776166.203222:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776166.331269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776166.331275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776166.331288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776166.331293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776166.331297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776166.331298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776166.331359:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776166.331363:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776166.331367:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776167.227229:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776167.227255:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776167.227257:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776167.227261:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776167.227263:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776167.227266:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776167.227268:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776167.227270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776167.227272:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776167.355258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776167.355262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776167.355266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776167.355267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776167.355270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776167.355271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776167.355318:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776167.355321:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776167.355324:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776168.251195:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776168.251200:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776168.251201:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776168.251206:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776168.251208:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776168.251210:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776168.251212:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776168.251214:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776168.251227:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776168.379274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776168.379280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776168.379285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776168.379286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776168.379289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776168.379289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776168.379348:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776168.379352:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776168.379356:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776169.083236:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307724038, at 4307724096 00000100:00000040:1.0:1775776169.083236:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307724038, at 4307724096 00000100:00000010:1.0:1775776169.083248:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e20c0. 00000100:00000010:0.0:1775776169.083251:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ba0. 00000100:00000001:1.0:1775776169.275228:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776169.275234:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776169.275236:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776169.275241:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776169.275244:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776169.275248:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776169.275251:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776169.275253:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776169.275256:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776169.403236:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776169.403242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776169.403246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776169.403248:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776169.403250:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776169.403253:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776169.403255:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776169.403257:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776169.403259:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776169.403261:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776169.403264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776169.403265:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776169.403266:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776169.403267:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776169.403267:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776169.403268:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776169.403269:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776169.403270:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776169.403323:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776169.403327:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776169.403330:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776170.299225:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776170.299231:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776170.299233:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776170.299237:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775776170.299238:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:0.0:1775776170.299244:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776170.299244:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776170.299246:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776170.299249:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775776170.299250:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776170.299256:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776170.299256:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776170.299259:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775776170.299262:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13052 00000100:00000001:1.0:1775776170.299264:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776170.299273:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776170.299280:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776170.299283:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776170.299287:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776170.299290:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776170.299293:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776170.299295:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776170.299299:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776170.299300:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776170.299303:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776170.299306:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776170.299308:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776170.299311:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776170.299313:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776170.299314:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776170.299315:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776170.299316:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776170.299318:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776170.299320:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776170.299321:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776170.299324:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776170.299325:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776170.299327:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776170.299329:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776170.299330:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776170.299337:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063800. 00000020:00000040:0.0:1775776170.299339:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776170.299342:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776170.299343:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776170.299345:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776170.299347:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776170.299350:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776170.299356:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000001:0.0:1775776170.299361:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776170.299367:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063800 x1862022710394240/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776170.299374:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776170.299377:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776170.299380:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000015d81d25] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063800 x1862022710394240/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776170.299395:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776170.299397:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776170.299399:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776170.299400:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776170.299401:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776170.299403:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776170.299404:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13052 00000400:00000200:0.0:1775776170.299407:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776170.299409:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776170.299411:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776170.299413:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776170.299415:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776170.299417:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776170.299418:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776170.299419:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776170.299420:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776170.299421:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776170.299422:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775776170.299423:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776170.299423:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1775776170.299427:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776170.299428:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1775776170.299429:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776170.299431:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776170.299432:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776170.299432:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1775776170.299435:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776170.299437:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775776170.299441:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776170.299442:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776170.299442:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e063800 x1862022710394240/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775776170.299445:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776170.299448:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776170.299452:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776170.299454:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776170.299458:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775776170.299459:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000008:00000001:0.0:1775776170.299462:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776170.299462:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776170.299466:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776170.299468:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:0.0:1775776170.299469:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060380. 00000020:00000040:0.0:1775776170.299471:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776170.299471:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776170.299475:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776170.299477:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776170.299477:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776170.299482:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776170.299482:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776170.299485:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776170.299485:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e063800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710394240:192.168.201.115@tcp:250:kworker.0 02000000:00000010:0.0:1775776170.299490:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6000. 00000100:00000001:0.0:1775776170.299493:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776170.299493:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00100000:0.0:1775776170.299499:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060380 x1862022710394368/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1775776170.299499:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776170.299503:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776170.299508:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1775776170.299511:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776170.299514:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776170.299515:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 00000100:00000040:0.0:1775776170.299519:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000614ace82] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060380 x1862022710394368/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775776170.299519:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776170.299530:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776170.299534:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776170.299536:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000100:00080000:0.0:1775776170.299537:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776170.299542:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776170.299543:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:1.0:1775776170.299543:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00080000:0.0:1775776170.299546:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776170.299550:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000200:1.0:1775776170.299552:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710394240, portal 25 00000100:00080000:0.0:1775776170.299554:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13052 00000100:00000001:1.0:1775776170.299556:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776170.299559:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574567424 : -123823134984192 : ffff8f622e063800) 00000400:00000200:0.0:1775776170.299562:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776170.299569:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000040:1.0:1775776170.299569:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e063800 x1862022710394240/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775776170.299573:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776170.299580:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776170.299585:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776170.299586:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776170.299589:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776170.299592:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776170.299594:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776170.299594:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776170.299596:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776170.299599:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000100:00000040:1.0:1775776170.299599:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000020:00000001:0.0:1775776170.299602:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775776170.299604:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000010:1.0:1775776170.299605:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3c38. 00000100:00000040:0.0:1775776170.299607:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776170.299609:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776170.299610:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000200:1.0:1775776170.299610:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710394240, offset 0 00000100:00000001:0.0:1775776170.299614:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776170.299616:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775776170.299618:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776170.299619:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776170.299621:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776170.299622:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 02000000:00000001:0.0:1775776170.299624:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776170.299626:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776170.299627:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776170.299629:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776170.299630:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776170.299632:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776170.299633:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000010:0.0:1775776170.299637:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062680. 00000020:00000040:0.0:1775776170.299640:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:1.0:1775776170.299642:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 02000000:00000001:0.0:1775776170.299643:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776170.299644:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776170.299647:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776170.299648:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776170.299649:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776170.299650:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000010:0.0:1775776170.299653:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7800. 00000400:00000200:1.0:1775776170.299656:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:0.0:1775776170.299658:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776170.299662:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00100000:0.0:1775776170.299663:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062680 x1862022710394496/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776170.299670:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:0.0:1775776170.299672:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776170.299676:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776170.299678:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453632:453632:256:4294967295] 192.168.201.115@tcp LPNI seq info [453557:453557:8:4294967295] 00000100:00000040:0.0:1775776170.299681:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000082c7985e] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062680 x1862022710394496/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776170.299689:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776170.299694:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776170.299696:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776170.299702:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776170.299706:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776170.299707:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000040:0.0:1775776170.299712:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776170.299717:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13063) 00000800:00000200:1.0:1775776170.299718:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776170.299725:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776170.299731:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776170.299735:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775776170.299739:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776170.299742:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000100:1.0:1775776170.299746:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776170.299750:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776170.299755:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776170.299759:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776170.299764:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3c38 00000400:00000010:1.0:1775776170.299766:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3c38. 00000100:00000001:1.0:1775776170.299771:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776170.299775:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e063800 x1862022710394240/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.299784:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776170.299788:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e063800 x1862022710394240/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776195 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.299795:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776170.299797:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776170.299802:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776170.299805:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776170.299808:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307729158 00000100:00000010:1.0:1775776170.299813:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e20c0. 00000100:00000001:1.0:1775776170.299818:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776170.299823:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307729158 00000100:00000010:1.0:1775776170.299826:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2b40. 00000100:00000001:1.0:1775776170.299829:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776170.299832:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776170.299834:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776170.299836:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776170.299840:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776170/real 1775776170] req@ffff8f622e063800 x1862022710394240/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776170.299848:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776170.299850:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776170.299852:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776170.299854:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776170.299856:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776170.299858:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776170.299862:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e063800 x1862022710394240/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776170.299870:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e063800 x1862022710394240/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.299876:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776170.299877:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776170.299879:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776170.299881:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776170.299884:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e063800 x1862022710394240/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.299891:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776170.299895:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e063800 x1862022710394240/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.299902:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776170.299904:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776170.299905:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776170.299906:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776170.299909:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776170.299911:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776170.299914:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776170.299917:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776170.299919:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776170.299924:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e063800 x1862022710394240/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776170.299931:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e063800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710394240:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776170.299935:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776170.299937:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776170.299939:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e063800 x1862022710394240/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.299951:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776170.299953:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776170.299955:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776170.299958:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776170.299960:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776170.299961:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776170.299963:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776170.299966:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:1.0:1775776170.299969:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776170.299970:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776170.299972:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e063800. 00000100:00000001:1.0:1775776170.299975:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776170.299976:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776170.299978:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776170.299984:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776170.299986:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776170.299987:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776170.299989:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776170.299990:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776170.299993:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e060380 x1862022710394368/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.299998:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776170.300000:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776170.300001:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776170.300003:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776170.300005:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776170.300008:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e060380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710394368:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776170.300011:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776170.300012:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776170.300014:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776170.300017:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776170.300019:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776170.300021:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776170.300025:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776170.300028:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776170.300031:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776170.300034:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710394368, portal 10 00000100:00000001:1.0:1775776170.300036:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776170.300037:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574553984 : -123823134997632 : ffff8f622e060380) 00000100:00000040:1.0:1775776170.300043:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e060380 x1862022710394368/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.300048:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776170.300051:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776170.300052:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3c38. 00000100:00000200:1.0:1775776170.300055:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710394368, offset 0 00000400:00000200:1.0:1775776170.300058:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776170.300061:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776170.300064:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776170.300067:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776170.300069:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776170.300072:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776170.300074:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776170.300076:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453633:453633:256:4294967295] 192.168.201.115@tcp LPNI seq info [453558:453558:8:4294967295] 00000400:00000200:1.0:1775776170.300080:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776170.300085:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776170.300089:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776170.300092:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776170.300096:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776170.300098:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775776170.300102:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776170.300104:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000100:1.0:1775776170.300108:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776170.300109:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776170.300112:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776170.300115:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776170.300118:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3c38 00000400:00000010:1.0:1775776170.300120:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3c38. 00000100:00000001:1.0:1775776170.300123:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776170.300126:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e060380 x1862022710394368/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.300151:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776170.300154:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e060380 x1862022710394368/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776195 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.300162:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776170.300163:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776170.300167:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776170.300170:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776170.300173:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776170.300175:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307725324 00000100:00000001:1.0:1775776170.300178:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307729158 00000100:00000001:1.0:1775776170.300192:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307725324 00000100:00000001:1.0:1775776170.300194:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307729158 00000100:00000001:1.0:1775776170.300196:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776170.300198:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776170.300200:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776170.300204:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776170/real 1775776170] req@ffff8f622e060380 x1862022710394368/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776170.300212:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776170.300213:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776170.300215:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776170.300217:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776170.300219:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776170.300221:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776170.300223:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e060380 x1862022710394368/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776170.300231:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e060380 x1862022710394368/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.300243:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776170.300244:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776170.300245:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776170.300247:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776170.300250:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e060380 x1862022710394368/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.300256:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776170.300259:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e060380 x1862022710394368/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.300265:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776170.300267:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776170.300268:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776170.300269:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776170.300271:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776170.300275:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776170.300278:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776170.300280:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776170.300282:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776170.300286:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e060380 x1862022710394368/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776170.300293:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e060380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710394368:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776170.300296:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776170.300299:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e062680 x1862022710394496/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.300305:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776170.300306:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776170.300309:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776170.300310:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776170.300312:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776170.300315:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e062680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710394496:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776170.300318:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776170.300319:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776170.300321:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776170.300322:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776170.300325:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776170.300327:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776170.300330:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776170.300332:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776170.300336:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776170.300338:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710394496, portal 10 00000100:00000001:1.0:1775776170.300340:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776170.300341:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574562944 : -123823134988672 : ffff8f622e062680) 00000100:00000040:1.0:1775776170.300347:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e062680 x1862022710394496/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.300353:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776170.300355:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776170.300357:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3c38. 00000100:00000200:1.0:1775776170.300359:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710394496, offset 0 00000400:00000200:1.0:1775776170.300363:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776170.300367:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776170.300371:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776170.300374:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776170.300376:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776170.300378:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776170.300380:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776170.300383:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453634:453634:256:4294967295] 192.168.201.115@tcp LPNI seq info [453559:453559:8:4294967295] 00000400:00000200:1.0:1775776170.300387:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776170.300392:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776170.300397:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776170.300400:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776170.300403:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776170.300405:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775776170.300409:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776170.300411:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000100:1.0:1775776170.300415:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776170.300416:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776170.300419:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776170.300422:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776170.300425:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3c38 00000400:00000010:1.0:1775776170.300426:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3c38. 00000100:00000001:1.0:1775776170.300429:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776170.300432:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e062680 x1862022710394496/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.300439:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776170.300442:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e062680 x1862022710394496/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776195 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.300448:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776170.300449:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776170.300452:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776170.300455:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776170.300459:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776170.300460:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307725324 00000100:00000001:1.0:1775776170.300462:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307729158 00000100:00000001:1.0:1775776170.300464:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307725324 00000100:00000001:1.0:1775776170.300466:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307729158 00000100:00000001:1.0:1775776170.300468:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776170.300470:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776170.300472:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776170.300475:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776170/real 1775776170] req@ffff8f622e062680 x1862022710394496/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776170.300484:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776170.300485:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776170.300487:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776170.300489:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776170.300491:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776170.300493:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776170.300496:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e062680 x1862022710394496/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776170.300503:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e062680 x1862022710394496/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.300510:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776170.300511:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776170.300513:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776170.300514:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776170.300518:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e062680 x1862022710394496/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.300524:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776170.300527:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e062680 x1862022710394496/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.300533:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776170.300536:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776170.300537:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776170.300538:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776170.300540:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776170.300542:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776170.300544:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776170.300547:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776170.300550:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776170.300554:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e062680 x1862022710394496/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776170.300560:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e062680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710394496:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776170.300564:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776170.300565:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776170.300568:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e060380 x1862022710394368/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.300573:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776170.300575:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776170.300577:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776170.300580:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776170.300581:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776170.300582:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776170.300584:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776170.300585:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:1.0:1775776170.300588:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776170.300589:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776170.300591:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e060380. 00000100:00000001:1.0:1775776170.300593:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776170.300595:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776170.300596:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776170.300598:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e062680 x1862022710394496/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776195 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776170.300604:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776170.300605:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776170.300606:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776170.300608:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776170.300609:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776170.300610:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776170.300612:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776170.300613:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:1.0:1775776170.300615:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776170.300616:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776170.300617:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e062680. 00000100:00000001:1.0:1775776170.300619:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776170.300620:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776170.300622:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776170.300625:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776170.300626:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776170.300628:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776170.300629:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776170.427244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776170.427249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776170.427253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776170.427254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776170.427257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776170.427258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776170.427324:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776170.427328:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776170.427331:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776171.323270:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776171.323276:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776171.323278:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776171.323283:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776171.323286:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776171.323289:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776171.323290:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776171.323293:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776171.323295:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776171.451303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776171.451312:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776171.451319:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776171.451320:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776171.451325:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776171.451327:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776171.451406:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776171.451412:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776171.451416:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776172.347213:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776172.347217:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776172.347219:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776172.347223:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776172.347226:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776172.347229:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776172.347231:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776172.347232:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776172.347234:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776172.475276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776172.475281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776172.475286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776172.475287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776172.475291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776172.475292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776172.475359:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776172.475363:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776172.475366:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776173.371237:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776173.371243:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776173.371245:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776173.371250:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776173.371254:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776173.371257:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776173.371259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776173.371262:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776173.371264:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776173.499220:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776173.499225:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776173.499231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776173.499232:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776173.499236:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776173.499237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776173.499295:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776173.499298:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776173.499300:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776174.203216:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307729158, at 4307729216 00000100:00000040:1.0:1775776174.203217:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307729158, at 4307729216 00000100:00000010:1.0:1775776174.203230:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2b40. 00000100:00000010:0.0:1775776174.203232:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e20c0. 00000100:00000001:1.0:1775776174.395244:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776174.395250:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776174.395252:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776174.395257:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776174.395260:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776174.395263:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776174.395266:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776174.395268:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776174.395279:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776174.523224:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776174.523229:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776174.523232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776174.523233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776174.523236:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776174.523237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776174.523296:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776174.523299:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776174.523303:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776175.419220:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776175.419225:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776175.419227:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776175.419231:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776175.419233:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776175.419235:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776175.419238:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776175.419239:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776175.419241:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776175.419343:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776175.419348:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776175.419350:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776175.419352:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776175.419356:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776175.419361:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13058 00000400:00000200:2.0:1775776175.419368:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776175.419373:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776175.419375:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776175.419378:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776175.419380:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776175.419382:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776175.419384:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776175.419387:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776175.419388:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776175.419390:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776175.419391:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776175.419394:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776175.419395:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776175.419397:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776175.419398:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776175.419400:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776175.419401:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776175.419402:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776175.419404:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776175.419405:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776175.419408:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776175.419409:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776175.419411:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776175.419413:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776175.419414:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776175.419421:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775776175.419423:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776175.419426:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776175.419427:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776175.419429:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776175.419430:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776175.419433:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776175.419438:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775776175.419443:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776175.419448:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710394624/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776175.419455:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776175.419458:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776175.419461:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710394624/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776175.419475:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776175.419477:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776175.419479:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776175.419480:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776175.419482:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776175.419483:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776175.419485:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13058 00000400:00000200:2.0:1775776175.419487:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776175.419489:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:1.0:1775776175.419491:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1775776175.419491:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776175.419493:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776175.419495:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776175.419495:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776175.419497:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776175.419498:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776175.419501:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776175.419502:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776175.419504:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000040:1.0:1775776175.419505:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344bb80 x1862022710394624/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775776175.419505:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776175.419508:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776175.419511:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776175.419514:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776175.419519:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:2.0:1775776175.419520:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776175.419521:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776175.419522:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775776175.419525:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:2.0:1775776175.419525:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776175.419527:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776175.419529:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776175.419530:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775776175.419534:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00100000:1.0:1775776175.419535:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710394624:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776175.419540:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1775776175.419541:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000001:1.0:1775776175.419544:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1775776175.419545:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776175.419546:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776175.419547:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776175.419549:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1775776175.419553:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:1.0:1775776175.419554:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776175.419557:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776175.419558:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776175.419561:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775776175.419563:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000008:00100000:2.0:1775776175.419566:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000010:1.0:1775776175.419567:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000008:00000001:2.0:1775776175.419571:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776175.419574:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000001:2.0:1775776175.419574:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776175.419579:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000100:00000200:1.0:1775776175.419580:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710394624, portal 25 00000100:00000001:1.0:1775776175.419583:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000040:2.0:1775776175.419583:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000001:1.0:1775776175.419586:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662550400 : -123823047001216 : ffff8f623344bb80) 02000000:00000001:2.0:1775776175.419586:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776175.419589:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776175.419591:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:1.0:1775776175.419592:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344bb80 x1862022710394624/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775776175.419593:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776175.419595:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776175.419600:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:1.0:1775776175.419601:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1775776175.419604:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776175.419605:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776175.419609:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3c38. 00000100:00000200:1.0:1775776175.419612:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710394624, offset 0 00000100:00100000:2.0:1775776175.419612:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710394752/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776175.419621:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:2.0:1775776175.419621:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776175.419624:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776175.419627:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710394752/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776175.419630:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776175.419637:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:2.0:1775776175.419637:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776175.419640:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776175.419642:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:1.0:1775776175.419643:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1775776175.419645:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776175.419647:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775776175.419648:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:2.0:1775776175.419651:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775776175.419653:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00080000:2.0:1775776175.419656:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13058 00000400:00000200:1.0:1775776175.419658:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:2.0:1775776175.419662:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776175.419664:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453635:453635:256:4294967295] 192.168.201.115@tcp LPNI seq info [453560:453560:8:4294967295] 00000400:00000200:2.0:1775776175.419667:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776175.419670:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1775776175.419673:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:2.0:1775776175.419674:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776175.419677:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776175.419678:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776175.419680:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776175.419682:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776175.419684:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000400:00000200:1.0:1775776175.419688:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000020:00000001:2.0:1775776175.419693:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:1.0:1775776175.419695:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000020:00000040:2.0:1775776175.419695:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776175.419698:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776175.419700:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000010:1.0:1775776175.419702:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000100:00000040:2.0:1775776175.419702:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776175.419706:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776175.419707:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776175.419709:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776175.419712:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000800:00000200:1.0:1775776175.419713:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00080000:2.0:1775776175.419716:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776175.419718:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776175.419719:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 02000000:00000001:2.0:1775776175.419719:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776175.419722:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.1:1775776175.419727:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:2.0:1775776175.419728:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776175.419730:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:1.1:1775776175.419732:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000008:00100000:2.0:1775776175.419733:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776175.419736:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776175.419738:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1775776175.419739:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000010:2.0:1775776175.419741:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000800:00000010:1.0:1775776175.419742:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000020:00000040:2.0:1775776175.419745:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776175.419747:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776175.419748:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776175.419749:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1775776175.419749:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776175.419752:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776175.419754:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776175.419757:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 02000000:00000010:2.0:1775776175.419758:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000400:00000200:1.0:1775776175.419761:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3c38 00000100:00000001:2.0:1775776175.419761:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776175.419764:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3c38. 00000100:00100000:2.0:1775776175.419767:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710394880/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776175.419771:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776175.419776:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344bb80 x1862022710394624/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776175.419785:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776175.419789:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776175.419791:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776175.419794:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710394880/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776175.419797:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344bb80 x1862022710394624/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776200 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776175.419812:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776175.419813:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776175.419814:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776175.419815:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:2.0:1775776175.419818:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776175.419820:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776175.419821:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776175.419823:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1775776175.419826:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:2.0:1775776175.419828:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13068) 00000100:00000001:1.0:1775776175.419829:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307734278 00000100:00000010:1.0:1775776175.419834:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2b40. 00000100:00000001:1.0:1775776175.419839:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776175.419844:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307734278 00000100:00000010:1.0:1775776175.419846:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2660. 00000100:00000001:1.0:1775776175.419848:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776175.419850:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776175.419851:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776175.419853:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776175.419855:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776175/real 1775776175] req@ffff8f623344bb80 x1862022710394624/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776175.419861:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776175.419862:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776175.419863:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776175.419864:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776175.419865:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776175.419867:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776175.419869:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344bb80 x1862022710394624/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776175.419875:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344bb80 x1862022710394624/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776175.419879:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776175.419880:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776175.419881:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776175.419882:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776175.419884:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344bb80 x1862022710394624/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776175.419888:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776175.419891:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344bb80 x1862022710394624/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776175.419895:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776175.419896:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776175.419897:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776175.419898:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776175.419899:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776175.419900:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776175.419902:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776175.419904:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776175.419906:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776175.419908:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344bb80 x1862022710394624/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776175.419913:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710394624:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776175.419916:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776175.419917:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776175.419918:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344bb80 x1862022710394624/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776175.419926:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776175.419928:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776175.419929:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776175.419931:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776175.419932:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776175.419933:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776175.419934:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776175.419936:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4f400. 02000000:00000001:1.0:1775776175.419938:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776175.419939:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776175.419940:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344bb80. 00000100:00000001:1.0:1775776175.419943:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776175.419944:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776175.419945:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776175.419950:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776175.419951:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776175.419952:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776175.419953:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776175.419953:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776175.419955:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233448700 x1862022710394752/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776175.419959:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776175.419960:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776175.419961:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776175.419963:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776175.419964:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776175.419965:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710394752:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776175.419967:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776175.419968:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776175.419969:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776175.419970:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776175.419971:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776175.419972:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776175.419974:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776175.419976:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776175.419978:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776175.419979:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710394752, portal 10 00000100:00000001:1.0:1775776175.419981:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776175.419981:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662536960 : -123823047014656 : ffff8f6233448700) 00000100:00000040:1.0:1775776175.419986:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233448700 x1862022710394752/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776175.419990:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776175.419991:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776175.419992:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3c38. 00000100:00000200:1.0:1775776175.419993:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710394752, offset 0 00000400:00000200:1.0:1775776175.419996:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776175.419998:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776175.420000:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776175.420001:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776175.420003:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776175.420004:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776175.420006:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776175.420008:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453636:453636:256:4294967295] 192.168.201.115@tcp LPNI seq info [453561:453561:8:4294967295] 00000400:00000200:1.0:1775776175.420010:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776175.420014:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776175.420016:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776175.420018:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776175.420020:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776175.420021:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775776175.420024:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776175.420025:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1775776175.420027:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776175.420029:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776175.420040:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776175.420045:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776175.420047:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3c38 00000400:00000010:1.0:1775776175.420048:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3c38. 00000100:00000001:1.0:1775776175.420050:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776175.420052:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233448700 x1862022710394752/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776175.420056:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776175.420058:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448700 x1862022710394752/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776200 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776175.420061:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776175.420062:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776175.420064:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776175.420067:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776175.420069:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776175.420070:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307730443 00000100:00000001:1.0:1775776175.420072:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307734278 00000100:00000001:1.0:1775776175.420073:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307730443 00000100:00000001:1.0:1775776175.420074:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307734278 00000100:00000001:1.0:1775776175.420076:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776175.420077:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776175.420078:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776175.420080:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776175/real 1775776175] req@ffff8f6233448700 x1862022710394752/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776175.420084:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776175.420084:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776175.420085:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776175.420086:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776175.420087:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776175.420088:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776175.420090:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233448700 x1862022710394752/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776175.420114:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233448700 x1862022710394752/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776175.420122:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776175.420123:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776175.420124:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776175.420125:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776175.420127:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233448700 x1862022710394752/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776175.420130:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776175.420132:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233448700 x1862022710394752/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776175.420151:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776175.420152:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776175.420153:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776175.420154:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776175.420155:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776175.420158:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776175.420159:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776175.420161:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776175.420162:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776175.420165:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233448700 x1862022710394752/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776175.420169:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710394752:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776175.420171:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776175.420173:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449c00 x1862022710394880/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776175.420176:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776175.420177:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776175.420178:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776175.420179:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776175.420180:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776175.420182:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710394880:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776175.420184:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776175.420185:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776175.420186:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776175.420187:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776175.420189:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776175.420190:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776175.420192:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776175.420193:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776175.420195:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776175.420197:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710394880, portal 10 00000100:00000001:1.0:1775776175.420198:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776175.420199:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662542336 : -123823047009280 : ffff8f6233449c00) 00000100:00000040:1.0:1775776175.420202:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449c00 x1862022710394880/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776175.420206:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776175.420208:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776175.420209:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3c38. 00000100:00000200:1.0:1775776175.420210:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710394880, offset 0 00000400:00000200:1.0:1775776175.420212:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776175.420214:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776175.420216:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776175.420218:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776175.420219:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776175.420221:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776175.420222:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776175.420224:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453637:453637:256:4294967295] 192.168.201.115@tcp LPNI seq info [453562:453562:8:4294967295] 00000400:00000200:1.0:1775776175.420226:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776175.420230:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776175.420233:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776175.420235:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776175.420237:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776175.420238:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775776175.420241:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776175.420242:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1775776175.420246:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776175.420247:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776175.420249:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776175.420251:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776175.420252:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3c38 00000400:00000010:1.0:1775776175.420253:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3c38. 00000100:00000001:1.0:1775776175.420255:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776175.420257:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449c00 x1862022710394880/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776175.420261:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776175.420262:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449c00 x1862022710394880/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776200 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776175.420266:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776175.420267:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776175.420268:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776175.420270:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776175.420272:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776175.420273:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307730444 00000100:00000001:1.0:1775776175.420274:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307734278 00000100:00000001:1.0:1775776175.420276:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307730444 00000100:00000001:1.0:1775776175.420277:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307734278 00000100:00000001:1.0:1775776175.420278:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776175.420279:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776175.420280:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776175.420282:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776175/real 1775776175] req@ffff8f6233449c00 x1862022710394880/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776175.420286:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776175.420287:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776175.420288:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776175.420289:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776175.420290:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776175.420291:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776175.420293:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449c00 x1862022710394880/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776175.420297:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449c00 x1862022710394880/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776175.420300:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776175.420301:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776175.420301:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776175.420302:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776175.420304:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449c00 x1862022710394880/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776175.420307:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776175.420309:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449c00 x1862022710394880/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776175.420312:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776175.420314:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776175.420315:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776175.420315:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776175.420316:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776175.420317:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776175.420319:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776175.420321:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776175.420322:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776175.420324:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449c00 x1862022710394880/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776175.420328:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710394880:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776175.420330:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776175.420331:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776175.420333:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233448700 x1862022710394752/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776175.420336:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776175.420337:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776175.420338:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776175.420339:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776175.420340:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776175.420340:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776175.420341:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776175.420342:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4cc00. 02000000:00000001:1.0:1775776175.420343:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776175.420344:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776175.420345:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233448700. 00000100:00000001:1.0:1775776175.420346:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776175.420347:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776175.420348:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776175.420349:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449c00 x1862022710394880/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776200 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776175.420353:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776175.420353:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776175.420354:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776175.420355:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776175.420356:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776175.420356:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776175.420357:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776175.420358:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4d400. 02000000:00000001:1.0:1775776175.420359:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776175.420360:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776175.420361:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449c00. 00000100:00000001:1.0:1775776175.420362:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776175.420363:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776175.420364:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776175.420366:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776175.420367:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776175.420368:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776175.420369:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776175.547241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776175.547246:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776175.547248:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776175.547250:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776175.547254:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776175.547256:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776175.547258:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776175.547260:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776175.547264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776175.547265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776175.547269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776175.547270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776175.547345:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776175.547348:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776175.547350:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776176.443260:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776176.443266:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776176.443268:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776176.443274:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776176.443278:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776176.443281:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776176.443284:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776176.443286:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776176.443289:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776176.571240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776176.571248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776176.571271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776176.571273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776176.571278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776176.571279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776176.571342:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776176.571346:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776176.571350:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776177.467205:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776177.467210:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776177.467212:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776177.467216:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776177.467219:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776177.467222:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776177.467223:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776177.467225:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776177.467227:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776177.595260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776177.595264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776177.595268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776177.595269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776177.595271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776177.595272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776177.595330:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776177.595333:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776177.595335:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776178.491240:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776178.491247:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776178.491249:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776178.491254:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776178.491257:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776178.491261:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776178.491263:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776178.491265:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776178.491268:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776178.619243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776178.619249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776178.619255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776178.619256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776178.619259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776178.619261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776178.619322:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776178.619326:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776178.619329:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776179.323219:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307734278, at 4307734336 00000100:00000040:1.0:1775776179.323221:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307734278, at 4307734336 00000100:00000010:0.0:1775776179.323234:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2b40. 00000100:00000010:1.0:1775776179.323236:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2660. 00000100:00000001:1.0:1775776179.515260:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776179.515266:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776179.515269:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776179.515274:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776179.515278:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776179.515281:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776179.515283:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776179.515285:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776179.515287:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776179.643289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776179.643298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776179.643305:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776179.643309:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776179.643312:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776179.643316:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776179.643320:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776179.643323:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776179.643325:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776179.643328:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776179.643332:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776179.643334:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776179.643335:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776179.643336:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776179.643338:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776179.643340:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776179.643342:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776179.643344:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776179.643405:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776179.643408:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776179.643411:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775776180.539244:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776180.539249:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1775776180.539255:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776180.539256:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776180.539260:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776180.539260:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1775776180.539266:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:2.0:1775776180.539266:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776180.539271:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776180.539272:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776180.539277:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776180.539280:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776180.539283:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775776180.539286:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13063 00000100:00000001:1.0:1775776180.539287:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776180.539298:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776180.539304:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776180.539307:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776180.539311:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776180.539314:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776180.539317:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776180.539320:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776180.539323:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776180.539325:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776180.539328:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776180.539330:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776180.539333:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776180.539335:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776180.539338:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776180.539340:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776180.539342:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776180.539344:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776180.539346:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776180.539349:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776180.539352:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776180.539355:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776180.539357:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776180.539360:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776180.539362:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776180.539364:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776180.539373:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775776180.539377:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776180.539380:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776180.539382:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776180.539385:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776180.539387:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776180.539391:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776180.539398:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775776180.539404:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776180.539412:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710395008/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776180.539423:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776180.539428:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776180.539432:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710395008/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776180.539450:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776180.539453:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776180.539455:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776180.539456:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776180.539457:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776180.539459:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776180.539460:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13063 00000400:00000200:2.0:1775776180.539463:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776180.539466:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776180.539468:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776180.539470:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776180.539471:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776180.539473:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776180.539475:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776180.539476:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776180.539477:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:1.0:1775776180.539478:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1775776180.539478:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775776180.539481:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000040:2.0:1775776180.539481:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:1.0:1775776180.539484:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1775776180.539484:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776180.539487:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776180.539491:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000040:1.0:1775776180.539492:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449500 x1862022710395008/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776180.539492:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776180.539495:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776180.539499:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776180.539503:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000001:1.0:1775776180.539509:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1775776180.539509:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776180.539512:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776180.539512:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776180.539516:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776180.539516:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776180.539519:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776180.539521:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776180.539522:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1775776180.539524:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710395008:192.168.201.115@tcp:250:kworker.0 02000000:00000001:2.0:1775776180.539528:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776180.539529:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1775776180.539531:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776180.539533:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776180.539535:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1775776180.539536:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775776180.539537:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:2.0:1775776180.539540:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776180.539541:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 00000100:00000001:2.0:1775776180.539542:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776180.539544:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776180.539548:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000400:00000200:1.0:1775776180.539549:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000040:2.0:1775776180.539553:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776180.539557:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776180.539558:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776180.539559:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000100:00000001:2.0:1775776180.539560:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776180.539563:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1775776180.539565:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 02000000:00000001:2.0:1775776180.539565:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776180.539571:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000200:1.0:1775776180.539572:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710395008, portal 25 00000100:00000001:1.0:1775776180.539576:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776180.539577:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662540544 : -123823047011072 : ffff8f6233449500) 00000100:00000001:2.0:1775776180.539577:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776180.539582:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449500 x1862022710395008/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776180.539587:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710395136/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776180.539591:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776180.539598:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:2.0:1775776180.539602:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1775776180.539603:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3c38. 00000100:00000001:2.0:1775776180.539606:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776180.539607:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710395008, offset 0 00000100:00000040:2.0:1775776180.539612:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710395136/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776180.539624:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776180.539626:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776180.539628:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00080000:2.0:1775776180.539629:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776180.539632:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776180.539634:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776180.539636:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775776180.539639:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00080000:2.0:1775776180.539643:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13063 00000400:00000200:1.0:1775776180.539647:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:2.0:1775776180.539650:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776180.539653:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776180.539656:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000400:00000200:1.0:1775776180.539659:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00080000:2.0:1775776180.539660:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1775776180.539664:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000040:2.0:1775776180.539666:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776180.539669:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:2.0:1775776180.539670:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776180.539674:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453638:453638:256:4294967295] 192.168.201.115@tcp LPNI seq info [453563:453563:8:4294967295] 00000100:00000040:2.0:1775776180.539677:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776180.539681:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:2.0:1775776180.539681:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776180.539685:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776180.539687:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776180.539690:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:1.0:1775776180.539693:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000020:00000040:2.0:1775776180.539694:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776180.539698:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:1.0:1775776180.539700:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000020:00000001:2.0:1775776180.539703:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000010:1.0:1775776180.539705:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000100:00000040:2.0:1775776180.539709:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776180.539711:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.0:1775776180.539715:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775776180.539715:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776180.539720:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000200:1.1:1775776180.539721:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000040:2.0:1775776180.539725:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776180.539730:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776180.539732:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776180.539734:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776180.539738:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776180.539740:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776180.539740:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776180.539742:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775776180.539746:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000008:00100000:2.0:1775776180.539746:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775776180.539749:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307739398 00000008:00000001:2.0:1775776180.539751:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776180.539754:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2660. 00000100:00000001:2.0:1775776180.539754:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776180.539760:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000100:00000001:1.0:1775776180.539762:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000020:00000040:2.0:1775776180.539765:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000100:00000001:1.0:1775776180.539769:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307739398 02000000:00000001:2.0:1775776180.539770:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776180.539774:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776180.539775:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2de0. 00000100:00000001:2.0:1775776180.539777:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775776180.539779:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1775776180.539779:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776180.539783:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776180.539784:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776180.539788:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776180.539789:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776180.539790:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:1.0:1775776180.539791:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776180.539797:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776180.539798:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776180.539800:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775776180.539802:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776180.539804:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776180.539805:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00100000:2.0:1775776180.539805:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710395264/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1775776180.539811:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344ad80 x1862022710395136/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776180.539826:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776180.539826:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776180.539829:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776180.539831:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1775776180.539831:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776180.539834:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776180.539836:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776180.539839:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710395264/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776180.539840:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710395136:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776180.539845:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776180.539847:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776180.539849:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776180.539853:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776180.539858:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d000. 00000100:00000001:2.0:1775776180.539859:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776180.539863:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776180.539864:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776180.539869:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:2.0:1775776180.539869:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776180.539874:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:1.0:1775776180.539875:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fe000. 00000100:00000040:2.0:1775776180.539880:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000010:1.0:1775776180.539882:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776180.539886:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710395136, portal 10 00000100:00000040:2.0:1775776180.539886:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13073) 00000100:00000001:1.0:1775776180.539890:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776180.539892:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662546816 : -123823047004800 : ffff8f623344ad80) 00000100:00000040:1.0:1775776180.539899:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344ad80 x1862022710395136/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776180.539906:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776180.539908:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776180.539910:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776180.539911:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710395136, offset 0 00000400:00000200:1.0:1775776180.539920:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776180.539923:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776180.539926:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776180.539928:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776180.539929:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776180.539931:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776180.539932:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776180.539934:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453639:453639:255:4294967295] 192.168.201.115@tcp LPNI seq info [453564:453564:7:4294967295] 00000400:00000200:1.0:1775776180.539937:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776180.539941:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776180.539943:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776180.539946:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230c00. 00000800:00000200:1.0:1775776180.539948:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776180.539951:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:1.0:1775776180.539952:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776180.539955:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776180.539957:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776180.539958:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307735563 00000100:00000001:1.0:1775776180.539960:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307739398 00000100:00000001:1.0:1775776180.539962:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307735563 00000100:00000001:1.0:1775776180.539963:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307739398 00000100:00000001:1.0:1775776180.539964:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776180.539966:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776180.539967:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776180.539969:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776180.539970:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776180.539971:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775776180.539973:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776180.539974:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776180.539974:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776180.539978:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449180 x1862022710395264/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776180.539984:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776180.539986:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776180.539987:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776180.539989:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776180.539990:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776180.539993:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710395264:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776180.540014:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776180.540016:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776180.540017:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776180.540018:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776180.540021:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776180.540023:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776180.540026:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776180.540028:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776180.540032:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776180.540034:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710395264, portal 10 00000100:00000001:1.0:1775776180.540036:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776180.540037:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662539648 : -123823047011968 : ffff8f6233449180) 00000100:00000040:1.0:1775776180.540045:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449180 x1862022710395264/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776180.540051:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776180.540054:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776180.540056:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f37f8. 00000100:00000200:1.0:1775776180.540058:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710395264, offset 0 00000400:00000200:1.0:1775776180.540062:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776180.540066:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776180.540068:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776180.540071:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776180.540074:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776180.540076:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776180.540078:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776180.540081:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453640:453640:254:4294967295] 192.168.201.115@tcp LPNI seq info [453565:453565:6:4294967295] 00000400:00000200:1.0:1775776180.540085:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776180.540090:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776180.540095:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776180.540098:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230000. 00000800:00000200:1.0:1775776180.540102:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776180.540105:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:1.0:1775776180.540108:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776180.540110:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776180.540113:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776180.540116:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307735563 00000100:00000001:1.0:1775776180.540118:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307739398 00000100:00000001:1.0:1775776180.540120:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307735563 00000100:00000001:1.0:1775776180.540122:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307739398 00000100:00000001:1.0:1775776180.540124:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776180.540126:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776180.540128:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776180.540129:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776180.540154:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776180.540155:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775776180.540157:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776180.540158:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776180.540160:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776180.540161:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776180.667209:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776180.667214:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776180.667220:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776180.667221:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776180.667224:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776180.667225:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776180.667282:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776180.667286:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776180.667290:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.0:1775776181.371217:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775776181.371229:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775776181.371233:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1775776181.371407:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000200:1.0:1775776181.371415:0:2404:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1775776181.371423:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775776181.371429:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776181.371432:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776181.563246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776181.563253:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776181.563255:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776181.563260:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776181.563264:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776181.563267:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776181.563270:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776181.563272:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776181.563274:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776181.691243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776181.691248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776181.691251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776181.691252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776181.691254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776181.691255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776181.691315:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776181.691319:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776181.691321:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776182.587241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776182.587246:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776182.587248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776182.587251:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776182.587255:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776182.587258:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776182.587260:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776182.587269:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776182.587271:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776182.715418:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776182.715428:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776182.715436:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776182.715437:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776182.715442:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776182.715443:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776182.715512:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776182.715517:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776182.715521:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776183.611247:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776183.611255:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776183.611257:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776183.611263:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776183.611267:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776183.611271:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776183.611274:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776183.611277:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776183.611280:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:3.0:1775776183.611479:0:2399:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:3.0:1775776183.611515:0:2399:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1775776183.611525:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776183.611529:0:2399:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:3.0:1775776183.611534:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776183.611537:0:2399:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:3.0:1775776183.611543:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775776183.611547:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:3.0:1775776183.611551:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775776183.611556:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775776183.611562:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3c38 00000400:00000010:3.0:1775776183.611564:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3c38. 00000100:00000001:3.0:1775776183.611570:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776183.611578:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6233449500 x1862022710395008/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776183.611603:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776183.611608:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449500 x1862022710395008/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776205 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776183.611614:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776183.611616:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775776183.611625:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775776183.611627:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230c00. 00000400:00000200:3.0:1775776183.611629:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775776183.611632:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775776183.611634:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:3.0:1775776183.611636:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:3.0:1775776183.611638:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776183.611643:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f623344ad80 x1862022710395136/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776183.611649:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776183.611651:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344ad80 x1862022710395136/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776205 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776183.611656:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776183.611658:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775776183.611661:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775776183.611663:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230000. 00000400:00000200:3.0:1775776183.611665:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775776183.611667:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775776183.611670:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f37f8 00000400:00000010:3.0:1775776183.611671:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f37f8. 00000100:00000001:3.0:1775776183.611673:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776183.611676:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6233449180 x1862022710395264/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776183.611682:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776183.611685:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449180 x1862022710395264/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776205 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776183.611690:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776183.611691:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776183.611742:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776183.611745:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776183.611748:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776183.611753:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776180/real 1775776183] req@ffff8f6233449180 x1862022710395264/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776183.611765:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776183.611767:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776183.611771:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776183.611774:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776183.611776:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776183.611779:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776183.611783:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449180 x1862022710395264/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776183.611793:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449180 x1862022710395264/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776183.611801:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776183.611802:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776183.611805:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776183.611807:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776183.611811:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449180 x1862022710395264/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776183.611819:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776183.611824:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449180 x1862022710395264/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776183.611833:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776183.611836:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776183.611837:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776183.611838:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776183.611841:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776183.611843:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776183.611847:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776183.611852:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776183.611854:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776183.611858:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449180 x1862022710395264/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776183.611865:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710395264:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776183.611870:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776183.611873:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776180/real 1775776183] req@ffff8f623344ad80 x1862022710395136/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776183.611882:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776183.611883:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776183.611885:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fe000. 00000400:00000200:1.0:1775776183.611886:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776183.611888:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776183.611889:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776183.611892:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344ad80 x1862022710395136/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776183.611899:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344ad80 x1862022710395136/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776183.611906:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776183.611907:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776183.611909:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776183.611910:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776183.611913:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344ad80 x1862022710395136/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776183.611919:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776183.611922:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344ad80 x1862022710395136/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776183.611929:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776183.611931:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776183.611932:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776183.611933:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776183.611935:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776183.611939:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776183.611942:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776183.611945:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776183.611948:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776183.611980:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344ad80 x1862022710395136/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776183.611989:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710395136:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776183.611994:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776183.611997:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776180/real 1775776183] req@ffff8f6233449500 x1862022710395008/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776183.612004:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776183.612005:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776183.612007:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776183.612009:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776183.612011:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776183.612013:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776183.612016:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449500 x1862022710395008/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776183.612025:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449500 x1862022710395008/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776183.612031:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776183.612033:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776183.612034:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776183.612035:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776183.612039:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449500 x1862022710395008/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776183.612044:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776183.612047:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449500 x1862022710395008/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776183.612057:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776183.612059:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776183.612061:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776183.612063:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776183.612064:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776183.612066:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776183.612068:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776183.612071:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776183.612073:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776183.612077:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449500 x1862022710395008/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776183.612084:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710395008:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776183.612087:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776183.612089:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776183.612092:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449180 x1862022710395264/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776183.612098:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776183.612102:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776183.612105:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776183.612109:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776183.612111:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776183.612113:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776183.612115:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776183.612117:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4d400. 02000000:00000001:1.0:1775776183.612121:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776183.612123:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776183.612125:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449180. 00000100:00000001:1.0:1775776183.612129:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776183.612130:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776183.612150:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776183.612153:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344ad80 x1862022710395136/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776183.612161:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776183.612163:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776183.612165:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776183.612168:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776183.612170:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776183.612171:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776183.612173:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776183.612174:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4e800. 02000000:00000001:1.0:1775776183.612177:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776183.612178:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776183.612179:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344ad80. 00000100:00000001:1.0:1775776183.612183:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776183.612185:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776183.612186:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776183.612189:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449500 x1862022710395008/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776205 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776183.612194:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776183.612196:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776183.612197:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776183.612199:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776183.612200:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776183.612201:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776183.612203:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776183.612204:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4c000. 02000000:00000001:1.0:1775776183.612206:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776183.612207:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776183.612208:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449500. 00000100:00000001:1.0:1775776183.612210:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776183.612211:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776183.612213:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776183.612220:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776183.612221:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776183.612223:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776183.612224:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776183.739252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776183.739258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776183.739263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776183.739264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776183.739268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776183.739269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776183.739340:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776183.739345:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776183.739348:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776184.443253:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307739398, at 4307739456 00000100:00000040:1.0:1775776184.443253:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307739398, at 4307739456 00000100:00000010:0.0:1775776184.443267:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2660. 00000100:00000010:1.0:1775776184.443269:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2de0. 00000100:00000001:1.0:1775776184.635269:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776184.635275:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776184.635277:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776184.635282:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776184.635286:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776184.635289:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776184.635292:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776184.635293:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776184.635296:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776184.764164:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776184.764171:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776184.764178:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776184.764179:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776184.764183:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776184.764184:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776184.764236:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776184.764240:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776184.764243:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776185.659329:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776185.659344:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776185.659347:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776185.659352:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776185.659357:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776185.659365:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13068 00000400:00000200:3.0:1775776185.659377:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776185.659385:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776185.659389:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776185.659392:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776185.659396:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776185.659399:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776185.659401:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776185.659406:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776185.659409:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776185.659412:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776185.659415:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776185.659418:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776185.659420:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776185.659423:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776185.659425:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776185.659427:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776185.659429:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776185.659432:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776185.659435:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776185.659438:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776185.659442:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776185.659445:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776185.659448:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776185.659451:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776185.659453:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776185.659466:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775776185.659470:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776185.659474:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776185.659476:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776185.659479:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776185.659482:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776185.659487:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776185.659497:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000100:00000001:3.0:1775776185.659506:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776185.659517:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710395392/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776185.659530:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776185.659546:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776185.659552:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710395392/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776185.659573:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776185.659576:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776185.659580:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776185.659581:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776185.659583:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776185.659586:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776185.659588:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13068 00000400:00000200:3.0:1775776185.659592:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776185.659595:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776185.659597:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776185.659599:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776185.659602:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776185.659604:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776185.659606:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776185.659608:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776185.659609:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776185.659611:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776185.659613:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776185.659615:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776185.659617:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776185.659620:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776185.659621:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776185.659623:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776185.659624:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776185.659626:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776185.659629:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776185.659633:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776185.659635:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776185.659640:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776185.659643:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776185.659644:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776185.659649:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776185.659652:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776185.659654:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776185.659657:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000020:00000040:3.0:1775776185.659660:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776185.659662:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776185.659664:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776185.659665:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776185.659667:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776185.659668:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776185.659672:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000100:00000001:3.0:1775776185.659675:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776185.659679:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710395520/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776185.659686:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776185.659688:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776185.659691:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710395520/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776185.659699:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776185.659701:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776185.659704:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776185.659706:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776185.659707:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776185.659709:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776185.659711:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13068 00000400:00000200:3.0:1775776185.659715:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776185.659718:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776185.659720:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776185.659723:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776185.659725:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776185.659726:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776185.659728:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776185.659730:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776185.659731:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776185.659732:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776185.659734:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776185.659736:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776185.659738:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776185.659740:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776185.659751:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776185.659753:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776185.659755:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776185.659758:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776185.659760:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776185.659762:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776185.659764:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776185.659766:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776185.659768:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776185.659770:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776185.659772:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776185.659774:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776185.659776:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776185.659779:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775776185.659781:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776185.659784:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776185.659785:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776185.659787:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776185.659789:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776185.659790:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776185.659793:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000100:00000001:3.0:1775776185.659796:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776185.659800:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710395648/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776185.659807:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776185.659809:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776185.659813:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710395648/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776185.659819:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776185.659822:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776185.659826:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776185.659829:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776185.659832:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776185.659837:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13078) 00000100:00000001:1.0:1775776185.659998:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776185.660003:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776185.660006:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776185.660014:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71500 x1862022710395392/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.660022:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776185.660025:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776185.660027:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 02000000:00000001:1.0:1775776185.660029:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776185.660034:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776185.660035:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:1.0:1775776185.660037:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776185.660038:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776185.660044:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710395392:192.168.201.115@tcp:250:kworker.0 00000100:00100000:2.0:1775776185.660044:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776185.660049:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776185.660051:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1775776185.660054:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775776185.660057:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1775776185.660057:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776185.660061:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775776185.660062:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776185.660068:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1775776185.660070:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776185.660074:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776185.660077:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776185.660083:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776185.660088:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776185.660097:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776185.660106:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710395392, portal 25 00000100:00000001:1.0:1775776185.660109:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776185.660112:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849009408 : -123823860542208 : ffff8f6202c71500) 00000100:00000040:1.0:1775776185.660118:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71500 x1862022710395392/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.660126:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776185.660129:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776185.660131:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776185.660158:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710395392, offset 0 00000400:00000200:1.0:1775776185.660167:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776185.660177:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776185.660185:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776185.660192:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776185.660197:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776185.660202:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776185.660205:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776185.660210:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453641:453641:256:4294967295] 192.168.201.115@tcp LPNI seq info [453566:453566:8:4294967295] 00000400:00000200:1.0:1775776185.660214:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776185.660225:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776185.660232:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776185.660238:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776185.660246:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776185.660250:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775776185.660257:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776185.660260:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775776185.660265:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776185.660268:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776185.660273:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776185.660278:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776185.660285:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776185.660287:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776185.660293:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776185.660297:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71500 x1862022710395392/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.660307:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776185.660310:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71500 x1862022710395392/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776210 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.660318:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776185.660320:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776185.660326:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776185.660331:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776185.660334:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307744519 00000100:00000010:1.0:1775776185.660340:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2de0. 00000100:00000001:1.0:1775776185.660348:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776185.660355:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307744519 00000100:00000010:1.0:1775776185.660358:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2300. 00000100:00000001:1.0:1775776185.660361:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776185.660365:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776185.660368:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776185.660370:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776185.660374:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776185/real 1775776185] req@ffff8f6202c71500 x1862022710395392/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776185.660385:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776185.660387:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776185.660389:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776185.660392:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776185.660394:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776185.660397:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776185.660400:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71500 x1862022710395392/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776185.660408:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71500 x1862022710395392/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.660415:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776185.660416:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776185.660419:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776185.660421:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776185.660425:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71500 x1862022710395392/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.660433:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776185.660437:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71500 x1862022710395392/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.660444:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776185.660447:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776185.660449:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776185.660450:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776185.660452:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776185.660455:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776185.660459:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776185.660464:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776185.660466:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776185.660470:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71500 x1862022710395392/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776185.660477:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710395392:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776185.660482:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776185.660485:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70a80 x1862022710395520/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.660491:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776185.660493:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776185.660495:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776185.660496:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776185.660498:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776185.660501:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710395520:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776185.660504:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776185.660506:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776185.660508:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776185.660510:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776185.660513:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776185.660515:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776185.660519:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776185.660521:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776185.660524:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776185.660527:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710395520, portal 10 00000100:00000001:1.0:1775776185.660529:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776185.660530:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849006720 : -123823860544896 : ffff8f6202c70a80) 00000100:00000040:1.0:1775776185.660537:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70a80 x1862022710395520/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.660543:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776185.660545:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776185.660547:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776185.660550:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710395520, offset 0 00000400:00000200:1.0:1775776185.660553:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776185.660557:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776185.660561:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776185.660564:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776185.660566:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776185.660568:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776185.660570:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776185.660573:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453642:453642:256:4294967295] 192.168.201.115@tcp LPNI seq info [453567:453567:8:4294967295] 00000400:00000200:1.0:1775776185.660577:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776185.660582:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776185.660586:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776185.660590:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776185.660594:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776185.660596:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775776185.660601:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776185.660603:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775776185.660607:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776185.660608:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776185.660612:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776185.660616:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776185.660618:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776185.660620:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776185.660622:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776185.660626:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70a80 x1862022710395520/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.660633:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776185.660636:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70a80 x1862022710395520/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776210 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.660642:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776185.660643:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776185.660646:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776185.660650:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776185.660653:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776185.660655:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307740684 00000100:00000001:1.0:1775776185.660658:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307744519 00000100:00000001:1.0:1775776185.660660:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307740684 00000100:00000001:1.0:1775776185.660662:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307744519 00000100:00000001:1.0:1775776185.660664:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776185.660666:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776185.660668:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776185.660671:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776185/real 1775776185] req@ffff8f6202c70a80 x1862022710395520/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776185.660680:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776185.660681:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776185.660682:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776185.660685:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776185.660686:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776185.660689:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776185.660692:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70a80 x1862022710395520/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776185.660698:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70a80 x1862022710395520/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.660703:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776185.660704:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776185.660705:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776185.660706:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776185.660709:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70a80 x1862022710395520/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.660715:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776185.660719:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70a80 x1862022710395520/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.660724:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776185.660727:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776185.660728:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776185.660729:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776185.660731:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776185.660737:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776185.660740:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776185.660743:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776185.660745:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776185.660748:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70a80 x1862022710395520/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776185.660754:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710395520:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776185.660757:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776185.660760:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70700 x1862022710395648/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.660772:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776185.660773:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776185.660775:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776185.660777:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776185.660779:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776185.660782:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710395648:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776185.660785:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776185.660788:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776185.660789:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776185.660791:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776185.660794:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776185.660795:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776185.660798:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776185.660800:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776185.660803:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776185.660805:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710395648, portal 10 00000100:00000001:1.0:1775776185.660807:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776185.660808:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849005824 : -123823860545792 : ffff8f6202c70700) 00000100:00000040:1.0:1775776185.660813:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70700 x1862022710395648/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.660818:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776185.660821:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776185.660822:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776185.660824:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710395648, offset 0 00000400:00000200:1.0:1775776185.660828:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776185.660831:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776185.660834:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776185.660837:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776185.660839:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776185.660842:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776185.660844:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776185.660847:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453643:453643:256:4294967295] 192.168.201.115@tcp LPNI seq info [453568:453568:8:4294967295] 00000400:00000200:1.0:1775776185.660850:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776185.660856:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776185.660859:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776185.660863:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776185.660866:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776185.660868:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775776185.660871:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776185.660875:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775776185.660878:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776185.660880:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776185.660882:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776185.660885:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776185.660887:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776185.660888:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776185.660891:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776185.660894:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70700 x1862022710395648/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.660900:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776185.660902:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70700 x1862022710395648/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776210 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.660908:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776185.660909:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776185.660911:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776185.660925:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776185.660929:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776185.660931:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307740684 00000100:00000001:1.0:1775776185.660933:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307744519 00000100:00000001:1.0:1775776185.660935:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307740684 00000100:00000001:1.0:1775776185.660937:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307744519 00000100:00000001:1.0:1775776185.660939:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776185.660941:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776185.660943:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776185.660947:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776185/real 1775776185] req@ffff8f6202c70700 x1862022710395648/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776185.660954:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776185.660955:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776185.660957:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776185.660960:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776185.660962:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776185.660964:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776185.660966:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70700 x1862022710395648/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776185.660973:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70700 x1862022710395648/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.660978:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776185.660980:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776185.660981:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776185.660982:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776185.660985:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70700 x1862022710395648/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.660991:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776185.660994:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70700 x1862022710395648/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.661000:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776185.661002:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776185.661003:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776185.661005:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776185.661006:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776185.661008:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776185.661010:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776185.661015:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776185.661017:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776185.661021:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70700 x1862022710395648/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776185.661027:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710395648:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776185.661030:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776185.661032:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776185.661034:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71500 x1862022710395392/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.661041:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776185.661044:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776185.661046:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776185.661049:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776185.661052:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776185.661053:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776185.661056:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776185.661058:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34000. 02000000:00000001:1.0:1775776185.661063:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776185.661065:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776185.661068:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71500. 00000100:00000001:1.0:1775776185.661071:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776185.661072:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776185.661074:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776185.661077:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70a80 x1862022710395520/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.663030:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776185.663035:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776185.663039:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776185.663046:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776185.663049:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776185.663052:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776185.663055:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776185.663058:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35c00. 02000000:00000001:1.0:1775776185.663063:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776185.663065:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776185.663067:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70a80. 00000100:00000001:1.0:1775776185.663071:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776185.663073:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776185.663076:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776185.663086:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70700 x1862022710395648/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776210 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776185.663094:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776185.663096:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776185.663098:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776185.663100:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776185.663102:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776185.663103:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776185.663106:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776185.663107:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36c00. 02000000:00000001:1.0:1775776185.663110:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776185.663111:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776185.663112:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70700. 00000100:00000001:1.0:1775776185.663115:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776185.663116:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776185.663118:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776185.663128:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776185.663131:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776185.663153:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776185.663156:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776185.787304:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776185.787314:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776185.787317:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776185.787321:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776185.787326:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776185.787329:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776185.787331:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776185.787334:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776185.787338:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776185.787339:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776185.787344:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776185.787345:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776185.787414:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776185.787419:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776185.787422:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776186.683296:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776186.683306:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776186.683308:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776186.683315:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776186.683320:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776186.683325:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776186.683329:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776186.683332:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776186.683335:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776186.811277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776186.811283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776186.811288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776186.811290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776186.811293:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776186.811294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776186.811353:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776186.811357:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776186.811360:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776187.707235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776187.707242:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776187.707244:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776187.707251:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776187.707255:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776187.707259:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776187.707262:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776187.707264:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776187.707267:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776187.835274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776187.835282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776187.835288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776187.835290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776187.835294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776187.835295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776187.835355:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776187.835358:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776187.835361:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776188.731245:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776188.731253:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776188.731255:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776188.731261:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776188.731264:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776188.731269:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776188.731271:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776188.731273:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776188.731276:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776188.859225:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776188.859230:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776188.859236:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776188.859237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776188.859240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776188.859242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776188.859320:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776188.859324:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776188.859327:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776189.563297:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307744519, at 4307744576 00000100:00000040:1.0:1775776189.563297:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307744519, at 4307744576 00000100:00000010:1.0:1775776189.563315:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2300. 00000100:00000010:0.0:1775776189.563316:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2de0. 00000100:00000001:2.0:1775776189.755243:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776189.755249:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776189.755251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776189.755256:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776189.755259:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776189.755263:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776189.755266:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776189.755268:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776189.755270:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776189.883275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776189.883282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776189.883289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776189.883293:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776189.883295:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776189.883299:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776189.883301:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776189.883304:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776189.883305:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776189.883307:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776189.883310:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776189.883311:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776189.883312:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776189.883321:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776189.883322:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776189.883324:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776189.883332:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776189.883334:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776189.883452:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776189.883457:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776189.883460:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776190.779253:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776190.779260:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775776190.779263:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776190.779268:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775776190.779270:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1775776190.779273:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776190.779274:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776190.779279:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00100000:2.0:1775776190.779282:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776190.779288:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776190.779288:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13073 00000100:00000001:2.0:1775776190.779296:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776190.779301:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:3.0:1775776190.779302:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:2.0:1775776190.779305:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776190.779310:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776190.779311:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776190.779318:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776190.779322:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776190.779325:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776190.779328:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776190.779330:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776190.779333:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776190.779334:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776190.779336:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776190.779339:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776190.779342:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776190.779344:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776190.779346:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776190.779348:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776190.779349:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776190.779350:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776190.779352:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776190.779362:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776190.779364:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776190.779368:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776190.779371:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776190.779374:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776190.779376:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776190.779379:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776190.779391:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775776190.779395:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776190.779399:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776190.779401:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776190.779405:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776190.779409:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776190.779413:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776190.779423:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000001:3.0:1775776190.779431:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776190.779440:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710395776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776190.779448:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776190.779452:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776190.779455:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710395776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776190.779475:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776190.779478:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776190.779481:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776190.779483:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776190.779484:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776190.779488:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776190.779491:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13073 00000400:00000200:3.0:1775776190.779495:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776190.779497:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776190.779499:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776190.779502:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776190.779504:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776190.779506:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776190.779507:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776190.779510:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776190.779511:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776190.779512:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776190.779514:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:1.0:1775776190.779516:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1775776190.779516:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776190.779518:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776190.779520:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775776190.779521:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776190.779523:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776190.779523:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776190.779525:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776190.779527:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:1.0:1775776190.779529:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71180 x1862022710395776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775776190.779529:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776190.779531:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776190.779534:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776190.779536:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776190.779538:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776190.779540:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776190.779542:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776190.779543:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776190.779544:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1775776190.779545:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776190.779546:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776190.779549:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775776190.779550:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775776190.779552:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710395776:192.168.201.115@tcp:250:kworker.0 00000008:00000001:3.0:1775776190.779553:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776190.779555:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776190.779557:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776190.779562:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:3.0:1775776190.779562:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775776190.779564:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776190.779565:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776190.779567:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776190.779568:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1775776190.779568:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776190.779570:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776190.779571:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:1.0:1775776190.779573:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d000. 02000000:00000001:3.0:1775776190.779573:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776190.779575:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776190.779581:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000010:3.0:1775776190.779584:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000400:00000010:1.0:1775776190.779586:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000100:00000001:3.0:1775776190.779587:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776190.779591:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710395904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1775776190.779593:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000001:3.0:1775776190.779598:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776190.779599:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776190.779601:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710395776, portal 25 00000100:00000040:3.0:1775776190.779603:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710395904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.779604:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776190.779606:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849008512 : -123823860543104 : ffff8f6202c71180) 00000100:00000040:1.0:1775776190.779611:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71180 x1862022710395776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776190.779611:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776190.779613:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776190.779618:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1775776190.779618:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776190.779619:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776190.779620:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1775776190.779622:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776190.779623:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000400:00000010:1.0:1775776190.779625:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00080000:3.0:1775776190.779626:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13073 00000100:00000200:1.0:1775776190.779627:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710395776, offset 0 00000400:00000200:3.0:1775776190.779628:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776190.779631:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776190.779633:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1775776190.779634:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776190.779636:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776190.779637:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776190.779639:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776190.779640:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776190.779642:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:1.0:1775776190.779643:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000040:3.0:1775776190.779643:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776190.779645:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776190.779647:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776190.779648:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776190.779650:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000020:00000001:3.0:1775776190.779650:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776190.779653:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776190.779654:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776190.779656:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1775776190.779656:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776190.779658:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776190.779659:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776190.779660:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00080000:3.0:1775776190.779661:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776190.779663:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776190.779664:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776190.779665:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 02000000:00000001:3.0:1775776190.779666:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775776190.779668:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 02000000:00000001:3.0:1775776190.779668:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776190.779670:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776190.779672:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1775776190.779673:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453644:453644:256:4294967295] 192.168.201.115@tcp LPNI seq info [453569:453569:8:4294967295] 00000008:00000001:3.0:1775776190.779675:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776190.779676:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776190.779677:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000010:3.0:1775776190.779679:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775776190.779681:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776190.779683:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776190.779684:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776190.779685:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776190.779686:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1775776190.779688:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 02000000:00000001:3.0:1775776190.779688:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776190.779691:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:3.0:1775776190.779693:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776190.779695:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00100000:3.0:1775776190.779697:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710396032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:1.0:1775776190.779700:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000100:00000001:3.0:1775776190.779703:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776190.779705:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776190.779708:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776190.779708:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710396032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776190.779715:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776190.779716:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000100:00000040:3.0:1775776190.779718:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776190.779721:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775776190.779722:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776190.779724:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775776190.779726:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000100:00000040:3.0:1775776190.779727:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000100:1.0:1775776190.779730:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776190.779732:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13083) 00000800:00000010:1.0:1775776190.779733:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776190.779739:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776190.779744:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776190.779749:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776190.779751:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776190.779757:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776190.779761:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71180 x1862022710395776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.779770:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776190.779773:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71180 x1862022710395776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776215 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.779778:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776190.779779:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776190.779793:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776190.779797:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776190.779799:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307749638 00000100:00000010:1.0:1775776190.779803:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2300. 00000100:00000001:1.0:1775776190.779810:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776190.779816:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307749638 00000100:00000010:1.0:1775776190.779817:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e25a0. 00000100:00000001:1.0:1775776190.779820:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776190.779822:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776190.779833:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776190.779836:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776190.779840:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776190/real 1775776190] req@ffff8f6202c71180 x1862022710395776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776190.779847:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776190.779848:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776190.779850:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776190.779852:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776190.779853:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776190.779855:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776190.779857:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71180 x1862022710395776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776190.779866:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71180 x1862022710395776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.779870:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776190.779872:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776190.779874:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776190.779875:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776190.779877:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71180 x1862022710395776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.779881:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776190.779885:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71180 x1862022710395776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.779890:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776190.779891:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776190.779892:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776190.779893:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776190.779894:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776190.779897:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776190.779899:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776190.779902:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776190.779903:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776190.779906:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71180 x1862022710395776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776190.779911:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710395776:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776190.779914:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776190.779916:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776190.779917:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71180 x1862022710395776/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.779921:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776190.779923:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776190.779925:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776190.779927:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776190.779927:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776190.779928:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776190.779930:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776190.779931:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37800. 02000000:00000001:1.0:1775776190.779935:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776190.779936:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776190.779938:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71180. 00000100:00000001:1.0:1775776190.779940:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776190.779941:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776190.779942:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776190.779948:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776190.779949:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776190.779951:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776190.779951:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776190.779953:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776190.779954:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72a00 x1862022710395904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.779963:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776190.779964:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776190.779965:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776190.779967:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776190.779968:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776190.779969:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710395904:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776190.779971:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776190.779972:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776190.779973:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776190.779974:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776190.779975:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776190.779976:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776190.779979:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776190.779981:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776190.779983:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776190.779985:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710395904, portal 10 00000100:00000001:1.0:1775776190.779986:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776190.779987:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849014784 : -123823860536832 : ffff8f6202c72a00) 00000100:00000040:1.0:1775776190.779992:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72a00 x1862022710395904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.779996:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776190.779997:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776190.779998:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776190.780000:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710395904, offset 0 00000400:00000200:1.0:1775776190.780002:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776190.780004:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776190.780006:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776190.780008:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776190.780009:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776190.780011:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776190.780013:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776190.780015:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453645:453645:256:4294967295] 192.168.201.115@tcp LPNI seq info [453570:453570:8:4294967295] 00000400:00000200:1.0:1775776190.780018:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776190.780022:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776190.780025:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776190.780027:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776190.780029:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776190.780031:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775776190.780034:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776190.780035:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775776190.780038:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776190.780039:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776190.780041:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776190.780043:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776190.780044:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776190.780045:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776190.780047:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776190.780049:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72a00 x1862022710395904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.780053:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776190.780055:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72a00 x1862022710395904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776215 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.780059:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776190.780060:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776190.780062:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776190.780064:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776190.780066:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776190.780067:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307745803 00000100:00000001:1.0:1775776190.780068:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307749638 00000100:00000001:1.0:1775776190.780070:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307745803 00000100:00000001:1.0:1775776190.780071:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307749638 00000100:00000001:1.0:1775776190.780073:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776190.780074:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776190.780075:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776190.780077:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776190/real 1775776190] req@ffff8f6202c72a00 x1862022710395904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776190.780082:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776190.780083:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776190.780083:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776190.780085:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776190.780085:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776190.780087:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776190.780088:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72a00 x1862022710395904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776190.780093:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72a00 x1862022710395904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.780097:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776190.780097:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776190.780098:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776190.780103:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776190.780105:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72a00 x1862022710395904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.780109:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776190.780110:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72a00 x1862022710395904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.780115:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776190.780116:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776190.780117:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776190.780118:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776190.780119:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776190.780123:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776190.780125:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776190.780127:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776190.780128:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776190.780131:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72a00 x1862022710395904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776190.780157:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710395904:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776190.780159:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776190.780161:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72300 x1862022710396032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.780165:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776190.780166:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776190.780167:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776190.780169:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776190.780170:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776190.780171:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710396032:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776190.780173:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776190.780173:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776190.780174:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776190.780175:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776190.780178:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776190.780179:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776190.780181:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776190.780182:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776190.780184:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776190.780186:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710396032, portal 10 00000100:00000001:1.0:1775776190.780187:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776190.780188:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849012992 : -123823860538624 : ffff8f6202c72300) 00000100:00000040:1.0:1775776190.780192:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72300 x1862022710396032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.780196:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776190.780197:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776190.780199:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776190.780200:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710396032, offset 0 00000400:00000200:1.0:1775776190.780202:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776190.780205:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776190.780207:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776190.780208:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776190.780210:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776190.780211:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776190.780213:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776190.780215:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453646:453646:256:4294967295] 192.168.201.115@tcp LPNI seq info [453571:453571:8:4294967295] 00000400:00000200:1.0:1775776190.780217:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776190.780221:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776190.780223:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776190.780225:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776190.780227:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776190.780229:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775776190.780231:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776190.780233:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775776190.780235:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776190.780236:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776190.780238:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776190.780240:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776190.780242:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776190.780243:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776190.780245:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776190.780246:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72300 x1862022710396032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776215 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.780251:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776190.780253:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72300 x1862022710396032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776215 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.780257:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776190.780258:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776190.780259:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776190.780261:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776190.780263:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776190.780264:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307745804 00000100:00000001:1.0:1775776190.780266:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307749638 00000100:00000001:1.0:1775776190.780267:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307745804 00000100:00000001:1.0:1775776190.780268:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307749638 00000100:00000001:1.0:1775776190.780269:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776190.780270:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776190.780271:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776190.780273:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776190/real 1775776190] req@ffff8f6202c72300 x1862022710396032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776190.780277:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776190.780278:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776190.780279:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776190.780280:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776190.780281:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776190.780282:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776190.780284:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72300 x1862022710396032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776190.780288:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72300 x1862022710396032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.780291:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776190.780292:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776190.780293:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776190.780293:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776190.780295:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72300 x1862022710396032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.780298:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776190.780301:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72300 x1862022710396032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.780305:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776190.780307:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776190.780308:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776190.780308:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776190.780309:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776190.780310:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776190.780311:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776190.780313:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776190.780314:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776190.780316:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72300 x1862022710396032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776190.780319:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710396032:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776190.780321:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776190.780322:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776190.780324:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72a00 x1862022710395904/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.780328:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776190.780329:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776190.780330:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776190.780331:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776190.780332:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776190.780332:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776190.780333:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776190.780334:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35000. 02000000:00000001:1.0:1775776190.780336:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776190.780336:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776190.780337:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72a00. 00000100:00000001:1.0:1775776190.780339:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776190.780340:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776190.780341:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776190.780343:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72300 x1862022710396032/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776215 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776190.780347:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776190.780348:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776190.780348:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776190.780349:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776190.780350:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776190.780350:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776190.780351:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776190.780352:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37c00. 02000000:00000001:1.0:1775776190.780353:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776190.780353:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776190.780354:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72300. 00000100:00000001:1.0:1775776190.780355:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776190.780356:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776190.780357:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776190.780358:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776190.780359:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776190.780360:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776190.780361:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776190.907224:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776190.907229:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776190.907233:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776190.907234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776190.907236:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776190.907237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776190.907332:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776190.907338:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776190.907342:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776191.803254:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776191.803259:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776191.803260:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776191.803264:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776191.803267:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776191.803270:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776191.803272:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776191.803274:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776191.803276:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776191.931309:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776191.931318:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776191.931325:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776191.931327:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776191.931331:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776191.931332:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776191.931372:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776191.931377:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776191.931380:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776192.827235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776192.827242:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776192.827243:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776192.827249:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776192.827252:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776192.827255:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776192.827258:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776192.827260:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776192.827262:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776192.955286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776192.955294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776192.955301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776192.955302:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776192.955306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776192.955308:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776192.955374:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776192.955378:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776192.955382:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776193.851287:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776193.851296:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776193.851298:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776193.851305:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776193.851308:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776193.851312:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776193.851315:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776193.851317:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776193.851320:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776193.979269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776193.979278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776193.979285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776193.979286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776193.979290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776193.979291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776193.979356:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776193.979360:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776193.979364:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776194.683264:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307749638, at 4307749696 00000100:00000040:1.0:1775776194.683264:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307749638, at 4307749696 00000100:00000010:0.0:1775776194.683282:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2300. 00000100:00000010:1.0:1775776194.683284:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e25a0. 00000100:00000001:1.0:1775776194.875210:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776194.875227:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776194.875230:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776194.875237:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776194.875242:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776194.875247:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776194.875250:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776194.875253:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776194.875256:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776195.003301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776195.003309:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776195.003317:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776195.003318:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776195.003322:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776195.003323:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776195.003382:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776195.003387:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776195.003390:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776195.899272:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776195.899275:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776195.899282:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1775776195.899283:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776195.899285:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776195.899287:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1775776195.899293:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:3.0:1775776195.899294:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776195.899301:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776195.899302:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776195.899307:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775776195.899311:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13078 00000100:00000001:1.0:1775776195.899313:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:3.0:1775776195.899323:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776195.899326:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776195.899331:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776195.899338:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776195.899343:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776195.899346:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776195.899350:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776195.899353:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776195.899357:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776195.899361:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776195.899364:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776195.899367:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776195.899371:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776195.899374:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776195.899376:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776195.899379:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776195.899381:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776195.899383:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776195.899384:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776195.899387:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776195.899391:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776195.899393:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776195.899397:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776195.899400:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776195.899403:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776195.899405:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776195.899408:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776195.899419:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73b80. 00000020:00000040:3.0:1775776195.899422:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776195.899426:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776195.899428:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776195.899431:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776195.899434:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776195.899439:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776195.899448:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000001:3.0:1775776195.899455:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776195.899465:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73b80 x1862022710396160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776195.899483:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776195.899488:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776195.899494:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c52008f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73b80 x1862022710396160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776195.899517:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776195.899520:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776195.899524:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776195.899525:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776195.899527:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776195.899530:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776195.899532:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13078 00000400:00000200:3.0:1775776195.899536:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776195.899539:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776195.899542:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776195.899545:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776195.899548:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776195.899550:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776195.899553:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776195.899553:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776195.899555:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776195.899557:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:1.0:1775776195.899558:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1775776195.899559:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775776195.899562:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1775776195.899562:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776195.899566:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776195.899568:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:1.0:1775776195.899573:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73b80 x1862022710396160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776195.899574:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776195.899577:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776195.899580:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776195.899584:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776195.899589:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000001:1.0:1775776195.899595:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775776195.899595:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776195.899599:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776195.899599:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776195.899604:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776195.899606:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776195.899609:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776195.899610:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776195.899614:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776195.899615:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776195.899616:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776195.899622:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775776195.899623:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710396160:192.168.201.115@tcp:250:kworker.0 00000008:00000001:3.0:1775776195.899628:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776195.899630:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1775776195.899631:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776195.899638:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:3.0:1775776195.899638:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775776195.899642:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776195.899643:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776195.899645:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776195.899648:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776195.899650:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776195.899651:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776195.899653:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776195.899656:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776195.899657:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:3.0:1775776195.899661:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776195.899664:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 00000100:00100000:3.0:1775776195.899667:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710396288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1775776195.899668:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776195.899680:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776195.899680:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776195.899684:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776195.899686:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000100:00000040:3.0:1775776195.899688:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710396288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1775776195.899695:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000001:3.0:1775776195.899698:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776195.899701:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1775776195.899705:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710396160, portal 25 00000100:00080000:3.0:1775776195.899705:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776195.899708:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776195.899710:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1775776195.899710:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776195.899714:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849019264 : -123823860532352 : ffff8f6202c73b80) 00000100:00000001:3.0:1775776195.899715:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776195.899719:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13078 00000400:00000200:3.0:1775776195.899725:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:1.0:1775776195.899726:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73b80 x1862022710396160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775776195.899729:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776195.899748:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776195.899751:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776195.899754:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776195.899758:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776195.899759:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776195.899763:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000040:3.0:1775776195.899763:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776195.899765:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776195.899768:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000200:1.0:1775776195.899769:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710396160, offset 0 00000020:00000040:3.0:1775776195.899770:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776195.899773:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776195.899775:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776195.899777:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776195.899781:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776195.899782:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776195.899786:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776195.899788:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776195.899790:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776195.899792:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776195.899796:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000040:3.0:1775776195.899797:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776195.899802:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776195.899804:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776195.899806:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776195.899808:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776195.899811:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776195.899812:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:3.0:1775776195.899814:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776195.899816:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776195.899820:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776195.899822:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776195.899826:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000010:3.0:1775776195.899826:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775776195.899829:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776195.899832:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1775776195.899833:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 02000000:00000001:3.0:1775776195.899834:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776195.899837:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776195.899839:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1775776195.899841:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 02000000:00000001:3.0:1775776195.899842:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776195.899848:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 02000000:00000010:3.0:1775776195.899848:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775776195.899854:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776195.899858:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453647:453647:256:4294967295] 192.168.201.115@tcp LPNI seq info [453572:453572:8:4294967295] 00000100:00100000:3.0:1775776195.899862:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710396416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776195.899868:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776195.899877:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776195.899879:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776195.899886:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710396416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776195.899890:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:3.0:1775776195.899901:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776195.899903:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776195.899905:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776195.899910:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000010:1.0:1775776195.899912:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000100:00000040:3.0:1775776195.899913:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776195.899917:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.0:1775776195.899922:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776195.899923:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13088) 00000800:00000200:1.0:1775776195.899928:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775776195.899937:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776195.899940:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775776195.899945:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776195.899948:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776195.899955:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776195.899960:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776195.899966:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776195.899968:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776195.899975:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776195.899979:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73b80 x1862022710396160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.899989:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776195.899992:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73b80 x1862022710396160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776220 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.899999:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776195.900001:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776195.900007:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776195.900011:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776195.900014:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307754758 00000100:00000010:1.0:1775776195.900019:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e25a0. 00000100:00000001:1.0:1775776195.900026:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776195.900041:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307754758 00000100:00000010:1.0:1775776195.900046:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2120. 00000100:00000001:1.0:1775776195.900049:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776195.900053:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776195.900056:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776195.900058:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776195.900064:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776195/real 1775776195] req@ffff8f6202c73b80 x1862022710396160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776195.900074:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776195.900076:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776195.900078:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776195.900082:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776195.900084:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776195.900087:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776195.900090:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73b80 x1862022710396160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776195.900099:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73b80 x1862022710396160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.900106:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776195.900107:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776195.900110:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776195.900112:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776195.900115:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73b80 x1862022710396160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.900121:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776195.900126:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73b80 x1862022710396160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.900152:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776195.900156:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776195.900157:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776195.900158:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776195.900161:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776195.900163:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776195.900166:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776195.900171:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776195.900173:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776195.900178:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73b80 x1862022710396160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776195.900186:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710396160:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776195.900190:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776195.900192:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776195.900194:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73b80 x1862022710396160/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.900200:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776195.900204:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776195.900207:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776195.900210:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776195.900212:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776195.900213:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776195.900216:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776195.900218:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35800. 02000000:00000001:1.0:1775776195.900222:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776195.900223:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776195.900225:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73b80. 00000100:00000001:1.0:1775776195.900228:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776195.900229:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776195.900231:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776195.900239:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776195.900241:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776195.900243:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776195.900244:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776195.900245:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776195.900249:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72300 x1862022710396288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.900256:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776195.900257:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776195.900259:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776195.900261:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776195.900262:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776195.900265:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710396288:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776195.900268:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776195.900270:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776195.900272:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776195.900273:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776195.900276:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776195.900278:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776195.900282:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776195.900284:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776195.900288:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776195.900290:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710396288, portal 10 00000100:00000001:1.0:1775776195.900293:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776195.900294:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849012992 : -123823860538624 : ffff8f6202c72300) 00000100:00000040:1.0:1775776195.900300:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72300 x1862022710396288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.900307:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776195.900309:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776195.900311:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776195.900313:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710396288, offset 0 00000400:00000200:1.0:1775776195.900325:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776195.900330:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776195.900333:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776195.900336:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776195.900338:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776195.900341:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776195.900343:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776195.900346:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453648:453648:256:4294967295] 192.168.201.115@tcp LPNI seq info [453573:453573:8:4294967295] 00000400:00000200:1.0:1775776195.900350:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776195.900355:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776195.900360:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776195.900363:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776195.900366:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776195.900369:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775776195.900373:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776195.900375:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775776195.900379:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776195.900380:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776195.900383:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776195.900386:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776195.900389:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776195.900390:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776195.900393:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776195.900396:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72300 x1862022710396288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.900409:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776195.900411:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72300 x1862022710396288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776220 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.900417:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776195.900418:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776195.900421:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776195.900426:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776195.900429:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776195.900430:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307750924 00000100:00000001:1.0:1775776195.900433:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307754758 00000100:00000001:1.0:1775776195.900435:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307750924 00000100:00000001:1.0:1775776195.900438:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307754758 00000100:00000001:1.0:1775776195.900440:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776195.900442:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776195.900443:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776195.900446:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776195/real 1775776195] req@ffff8f6202c72300 x1862022710396288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776195.900454:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776195.900455:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776195.900457:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776195.900459:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776195.900460:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776195.900462:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776195.900466:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72300 x1862022710396288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776195.900473:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72300 x1862022710396288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.900479:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776195.900480:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776195.900482:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776195.900483:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776195.900487:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72300 x1862022710396288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.900492:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776195.900496:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72300 x1862022710396288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.900503:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776195.900505:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776195.900506:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776195.900508:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776195.900509:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776195.900514:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776195.900517:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776195.900520:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776195.900522:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776195.900526:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72300 x1862022710396288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776195.900534:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710396288:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776195.900538:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776195.900541:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72a00 x1862022710396416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.900546:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776195.900548:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776195.900550:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776195.900551:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776195.900553:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776195.900557:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710396416:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776195.900559:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776195.900561:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776195.900563:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776195.900564:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776195.900568:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776195.900569:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776195.900572:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776195.900575:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776195.900577:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776195.900580:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710396416, portal 10 00000100:00000001:1.0:1775776195.900582:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776195.900583:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849014784 : -123823860536832 : ffff8f6202c72a00) 00000100:00000040:1.0:1775776195.900588:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72a00 x1862022710396416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.900594:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776195.900596:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776195.900598:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776195.900600:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710396416, offset 0 00000400:00000200:1.0:1775776195.900603:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776195.900607:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776195.900610:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776195.900613:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776195.900615:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776195.900617:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776195.900620:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776195.900623:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453649:453649:256:4294967295] 192.168.201.115@tcp LPNI seq info [453574:453574:8:4294967295] 00000400:00000200:1.0:1775776195.900626:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776195.900631:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776195.900635:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776195.900638:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776195.900642:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776195.900645:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775776195.900649:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776195.900651:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775776195.900655:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776195.900656:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776195.900659:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776195.900662:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776195.900665:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776195.900667:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776195.900669:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776195.900672:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72a00 x1862022710396416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776220 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.900679:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776195.900681:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72a00 x1862022710396416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776220 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.900687:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776195.900689:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776195.900691:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776195.900694:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776195.900697:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776195.900699:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307750924 00000100:00000001:1.0:1775776195.900701:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307754758 00000100:00000001:1.0:1775776195.900703:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307750924 00000100:00000001:1.0:1775776195.900706:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307754758 00000100:00000001:1.0:1775776195.900707:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776195.900709:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776195.900711:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776195.900715:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776195/real 1775776195] req@ffff8f6202c72a00 x1862022710396416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776195.900722:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776195.900723:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776195.900725:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776195.900727:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776195.900729:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776195.900731:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776195.900733:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72a00 x1862022710396416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776195.900755:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72a00 x1862022710396416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.900761:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776195.900763:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776195.900765:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776195.900766:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776195.900769:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72a00 x1862022710396416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.900775:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776195.900778:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72a00 x1862022710396416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.900785:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776195.900787:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776195.900789:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776195.900790:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776195.900791:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776195.900794:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776195.900796:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776195.900799:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776195.900801:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776195.900805:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72a00 x1862022710396416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776195.900817:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710396416:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776195.900821:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776195.900822:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776195.900825:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72300 x1862022710396288/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.900832:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776195.900833:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776195.900835:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776195.900838:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776195.900848:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776195.900850:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776195.900856:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776195.900858:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35400. 02000000:00000001:1.0:1775776195.900860:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776195.900861:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776195.900863:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72300. 00000100:00000001:1.0:1775776195.900866:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776195.900867:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776195.900869:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776195.900872:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72a00 x1862022710396416/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776220 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776195.900878:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776195.900879:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776195.900880:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776195.900882:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776195.900884:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776195.900885:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776195.900887:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776195.900888:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37400. 02000000:00000001:1.0:1775776195.900890:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776195.900891:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776195.900893:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72a00. 00000100:00000001:1.0:1775776195.900895:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776195.900897:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776195.900898:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776195.900901:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776195.900903:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776195.900904:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776195.900906:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776196.027270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776196.027280:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776196.027282:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776196.027286:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776196.027292:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776196.027295:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776196.027302:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776196.027305:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776196.027309:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776196.027310:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776196.027314:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776196.027316:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776196.027447:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776196.027451:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776196.027454:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776196.923282:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776196.923290:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776196.923292:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776196.923300:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776196.923311:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776196.923315:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776196.923318:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776196.923321:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776196.923324:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776197.051290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776197.051296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776197.051301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776197.051302:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776197.051305:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776197.051306:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776197.051324:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776197.051329:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776197.051333:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776197.947265:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776197.947274:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776197.947276:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776197.947283:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776197.947288:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776197.947293:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776197.947296:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776197.947298:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776197.947301:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776198.075249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776198.075253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776198.075256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776198.075257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776198.075261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776198.075262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776198.075323:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776198.075326:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776198.075328:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776198.971246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776198.971252:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776198.971254:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776198.971258:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776198.971260:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776198.971263:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776198.971266:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776198.971268:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776198.971270:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776199.099257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776199.099261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776199.099265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776199.099266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776199.099269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776199.099270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776199.099339:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776199.099343:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776199.099346:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776199.803240:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307754758, at 4307754816 00000100:00000040:1.0:1775776199.803240:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307754758, at 4307754816 00000100:00000010:1.0:1775776199.803254:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2120. 00000100:00000010:0.0:1775776199.803257:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e25a0. 00000100:00000001:1.0:1775776199.995266:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776199.995271:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776199.995273:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776199.995277:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776199.995279:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776199.995282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776199.995284:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776199.995286:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776199.995287:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776200.123255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776200.123261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776200.123266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776200.123269:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776200.123270:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776200.123273:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776200.123276:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776200.123277:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776200.123278:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776200.123280:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776200.123282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776200.123283:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776200.123284:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776200.123285:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776200.123286:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776200.123286:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776200.123287:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776200.123288:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776200.123339:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776200.123343:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776200.123347:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776201.019270:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775776201.019274:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776201.019277:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776201.019279:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776201.019286:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:3.0:1775776201.019286:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776201.019290:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776201.019291:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776201.019293:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776201.019296:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775776201.019296:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776201.019299:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776201.019302:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776201.019303:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776201.019311:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13083 00000400:00000200:3.0:1775776201.019322:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776201.019331:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776201.019334:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776201.019339:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776201.019343:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776201.019345:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776201.019348:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776201.019352:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776201.019354:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776201.019357:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776201.019360:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776201.019363:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776201.019365:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776201.019368:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776201.019370:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776201.019372:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776201.019374:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776201.019377:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776201.019380:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776201.019383:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776201.019387:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776201.019391:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776201.019394:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776201.019397:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776201.019399:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776201.019411:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775776201.019414:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776201.019419:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776201.019421:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776201.019424:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776201.019427:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776201.019432:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776201.019442:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000001:3.0:1775776201.019450:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776201.019460:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710396544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776201.019471:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776201.019476:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776201.019482:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710396544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776201.019505:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776201.019508:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776201.019511:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776201.019512:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776201.019514:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776201.019517:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776201.019520:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13083 00000400:00000200:3.0:1775776201.019525:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776201.019529:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776201.019531:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776201.019532:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1775776201.019535:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776201.019537:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776201.019540:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776201.019540:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776201.019544:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776201.019547:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:1.0:1775776201.019549:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71180 x1862022710396544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775776201.019550:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776201.019552:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776201.019556:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776201.019559:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:1.0:1775776201.019560:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776201.019562:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775776201.019563:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776201.019566:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1775776201.019567:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776201.019569:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776201.019572:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776201.019574:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00100000:1.0:1775776201.019576:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710396544:192.168.201.115@tcp:250:kworker.0 00000100:00000001:3.0:1775776201.019579:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776201.019581:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1775776201.019584:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776201.019585:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1775776201.019587:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776201.019589:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776201.019590:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:1.0:1775776201.019592:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1775776201.019596:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000010:1.0:1775776201.019598:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776201.019600:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776201.019600:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776201.019603:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776201.019608:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776201.019609:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1775776201.019611:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 02000000:00000001:3.0:1775776201.019613:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776201.019617:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776201.019624:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710396544, portal 25 00000100:00000001:3.0:1775776201.019624:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776201.019627:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776201.019629:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849008512 : -123823860543104 : ffff8f6202c71180) 00000008:00100000:3.0:1775776201.019630:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000040:1.0:1775776201.019634:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71180 x1862022710396544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1775776201.019636:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776201.019641:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776201.019657:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776201.019661:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776201.019663:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776201.019665:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710396544, offset 0 00000400:00000200:1.0:1775776201.019670:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000010:3.0:1775776201.019675:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775776201.019678:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000400:00000200:1.0:1775776201.019679:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776201.019684:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776201.019686:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776201.019687:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:3.0:1775776201.019690:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1775776201.019693:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1775776201.019693:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776201.019696:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776201.019698:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776201.019703:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 02000000:00000010:3.0:1775776201.019703:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000400:00000200:1.0:1775776201.019706:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:3.0:1775776201.019709:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776201.019711:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453650:453650:256:4294967295] 192.168.201.115@tcp LPNI seq info [453575:453575:8:4294967295] 00000100:00100000:3.0:1775776201.019716:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710396672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776201.019717:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776201.019727:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775776201.019728:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:3.0:1775776201.019732:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776201.019737:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776201.019740:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710396672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:1.0:1775776201.019742:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776201.019751:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776201.019753:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775776201.019758:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775776201.019761:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776201.019762:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000100:00080000:3.0:1775776201.019766:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1775776201.019767:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776201.019771:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000100:00080000:3.0:1775776201.019772:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776201.019775:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1775776201.019777:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:3.0:1775776201.019777:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776201.019782:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775776201.019784:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776201.019788:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000100:00080000:3.0:1775776201.019788:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13083 00000400:00000010:1.0:1775776201.019791:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000400:00000200:3.0:1775776201.019794:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776201.019796:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:3.0:1775776201.019799:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000200:1.0:1775776201.019801:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71180 x1862022710396544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775776201.019803:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776201.019811:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776201.019813:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1775776201.019816:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776201.019817:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71180 x1862022710396544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776201.019821:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776201.019825:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776201.019826:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776201.019831:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776201.019831:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776201.019837:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000001:3.0:1775776201.019837:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776201.019839:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000100:00000001:1.0:1775776201.019840:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307759878 00000020:00000001:3.0:1775776201.019842:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000010:1.0:1775776201.019845:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2120. 00000020:00000040:3.0:1775776201.019846:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776201.019850:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776201.019853:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000001:1.0:1775776201.019854:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000040:3.0:1775776201.019857:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776201.019860:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307759878 00000100:00000001:3.0:1775776201.019860:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776201.019863:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000010:1.0:1775776201.019864:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2f60. 00000100:00000001:3.0:1775776201.019868:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:1.0:1775776201.019869:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776201.019872:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1775776201.019873:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00000001:1.0:1775776201.019876:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776201.019879:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776201.019880:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776201.019884:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776201/real 1775776201] req@ffff8f6202c71180 x1862022710396544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776201.019885:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776201.019890:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:1.0:1775776201.019895:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1775776201.019895:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:1.0:1775776201.019897:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776201.019900:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 02000000:00000001:3.0:1775776201.019900:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776201.019904:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776201.019905:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776201.019907:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1775776201.019908:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000200:1.0:1775776201.019911:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71180 x1862022710396544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775776201.019912:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776201.019917:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776201.019921:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776201.019924:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71180 x1862022710396544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1775776201.019927:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000100:00000001:1.0:1775776201.019932:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:3.0:1775776201.019933:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000100:00000001:1.0:1775776201.019934:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776201.019938:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1775776201.019939:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1775776201.019940:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775776201.019942:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776201.019944:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71180 x1862022710396544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776201.019946:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776201.019949:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775776201.019952:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1775776201.019952:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776201.019958:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71180 x1862022710396544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1775776201.019959:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000100:00000001:1.0:1775776201.019965:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1775776201.019966:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776201.019968:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776201.019970:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776201.019971:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776201.019973:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776201.019975:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:3.0:1775776201.019975:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710396800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775776201.019979:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776201.019985:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776201.019988:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1775776201.019993:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1775776201.019994:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71180 x1862022710396544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776201.019997:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776201.020003:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710396544:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776201.020007:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776201.020009:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776201.020011:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71180 x1862022710396544/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776201.020014:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710396800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776201.020031:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776201.020033:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776201.020036:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d000. 00000100:00000001:3.0:1775776201.020037:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776201.020039:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776201.020041:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000040:3.0:1775776201.020041:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000040:1.0:1775776201.020043:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776201.020046:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000040:3.0:1775776201.020049:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 02000000:00000010:1.0:1775776201.020050:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34800. 02000000:00000001:1.0:1775776201.020055:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:3.0:1775776201.020055:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776201.020059:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 02000000:00000001:1.0:1775776201.020064:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776201.020065:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71180. 00000100:00000040:3.0:1775776201.020065:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13093) 00000100:00000001:1.0:1775776201.020070:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776201.020071:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776201.020073:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776201.020080:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776201.020081:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776201.020083:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776201.020084:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776201.020085:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776201.020088:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70700 x1862022710396672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776201.020092:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776201.020093:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776201.020094:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776201.020096:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776201.020097:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776201.020098:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710396672:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776201.020100:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776201.020101:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776201.020102:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776201.020104:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776201.020105:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776201.020107:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776201.020109:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776201.020111:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776201.020113:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776201.020115:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710396672, portal 10 00000100:00000001:1.0:1775776201.020116:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776201.020117:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849005824 : -123823860545792 : ffff8f6202c70700) 00000100:00000040:1.0:1775776201.020121:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70700 x1862022710396672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776201.020125:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776201.020127:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776201.020128:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776201.020130:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710396672, offset 0 00000400:00000200:1.0:1775776201.020132:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776201.020150:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776201.020153:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776201.020156:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776201.020158:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776201.020161:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776201.020163:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776201.020166:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453651:453651:256:4294967295] 192.168.201.115@tcp LPNI seq info [453576:453576:8:4294967295] 00000400:00000200:1.0:1775776201.020170:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776201.020175:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776201.020180:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776201.020183:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776201.020187:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776201.020190:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775776201.020194:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776201.020197:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1775776201.020201:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776201.020202:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776201.020205:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776201.020207:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776201.020209:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776201.020210:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776201.020212:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776201.020215:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70700 x1862022710396672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776201.020220:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776201.020221:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70700 x1862022710396672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776201.020225:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776201.020227:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776201.020228:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776201.020231:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776201.020233:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776201.020235:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307756044 00000100:00000001:1.0:1775776201.020237:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307759878 00000100:00000001:1.0:1775776201.020238:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307756044 00000100:00000001:1.0:1775776201.020240:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307759878 00000100:00000001:1.0:1775776201.020241:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776201.020242:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776201.020243:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776201.020245:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776201/real 1775776201] req@ffff8f6202c70700 x1862022710396672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776201.020249:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776201.020250:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776201.020251:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776201.020252:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776201.020253:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776201.020255:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776201.020256:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70700 x1862022710396672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776201.020261:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70700 x1862022710396672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776201.020268:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776201.020269:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776201.020270:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776201.020271:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776201.020273:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70700 x1862022710396672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776201.020276:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776201.020278:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70700 x1862022710396672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776201.020281:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776201.020283:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776201.020284:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776201.020284:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776201.020285:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776201.020289:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776201.020290:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776201.020292:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776201.020293:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776201.020295:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70700 x1862022710396672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776201.020300:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710396672:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776201.020302:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776201.020304:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70a80 x1862022710396800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776201.020307:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776201.020308:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776201.020310:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776201.020311:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776201.020312:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776201.020313:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710396800:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776201.020316:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776201.020317:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776201.020317:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776201.020318:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776201.020320:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776201.020321:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776201.020323:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776201.020325:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776201.020326:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776201.020328:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710396800, portal 10 00000100:00000001:1.0:1775776201.020329:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776201.020330:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849006720 : -123823860544896 : ffff8f6202c70a80) 00000100:00000040:1.0:1775776201.020333:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70a80 x1862022710396800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776201.020337:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776201.020338:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776201.020339:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776201.020341:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710396800, offset 0 00000400:00000200:1.0:1775776201.020343:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776201.020345:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776201.020347:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776201.020348:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776201.020350:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776201.020351:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776201.020353:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776201.020355:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453652:453652:256:4294967295] 192.168.201.115@tcp LPNI seq info [453577:453577:8:4294967295] 00000400:00000200:1.0:1775776201.020357:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776201.020361:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776201.020364:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776201.020366:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776201.020368:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776201.020370:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775776201.020372:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776201.020374:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1775776201.020376:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776201.020377:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776201.020379:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776201.020381:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776201.020389:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776201.020390:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776201.020392:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776201.020394:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70a80 x1862022710396800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776201.020398:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776201.020400:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70a80 x1862022710396800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776201.020404:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776201.020406:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776201.020408:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776201.020411:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776201.020414:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776201.020416:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307756044 00000100:00000001:1.0:1775776201.020419:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307759878 00000100:00000001:1.0:1775776201.020421:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307756044 00000100:00000001:1.0:1775776201.020423:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307759878 00000100:00000001:1.0:1775776201.020425:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776201.020427:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776201.020429:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776201.020433:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776201/real 1775776201] req@ffff8f6202c70a80 x1862022710396800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776201.020440:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776201.020442:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776201.020443:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776201.020446:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776201.020447:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776201.020449:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776201.020452:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70a80 x1862022710396800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776201.020461:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70a80 x1862022710396800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776201.020467:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776201.020469:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776201.020470:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776201.020471:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776201.020474:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70a80 x1862022710396800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776201.020478:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776201.020480:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70a80 x1862022710396800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776201.020484:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776201.020485:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776201.020486:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776201.020486:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776201.020487:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776201.020488:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776201.020490:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776201.020491:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776201.020492:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776201.020495:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70a80 x1862022710396800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776201.020503:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710396800:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776201.020507:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776201.020508:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776201.020511:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70700 x1862022710396672/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776201.020517:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776201.020518:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776201.020520:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776201.020522:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776201.020524:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776201.020525:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776201.020527:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776201.020529:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37000. 02000000:00000001:1.0:1775776201.020531:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776201.020533:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776201.020534:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70700. 00000100:00000001:1.0:1775776201.020537:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776201.020538:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776201.020539:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776201.020542:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70a80 x1862022710396800/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776201.020548:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776201.020549:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776201.020550:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776201.020551:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776201.020552:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776201.020553:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776201.020554:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776201.020555:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34400. 02000000:00000001:1.0:1775776201.020556:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776201.020557:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776201.020558:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70a80. 00000100:00000001:1.0:1775776201.020560:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776201.020560:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776201.020561:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776201.020564:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776201.020565:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776201.020566:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776201.020567:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776201.147280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776201.147288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776201.147296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776201.147298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776201.147301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776201.147303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776201.147368:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776201.147372:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776201.147375:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776202.043292:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776202.043301:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776202.043303:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776202.043311:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776202.043315:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776202.043321:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776202.043324:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776202.043326:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776202.043329:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776202.171288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776202.171294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776202.171301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776202.171302:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776202.171307:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776202.171309:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776202.171370:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776202.171375:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776202.171379:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776203.067261:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776203.067268:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776203.067275:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776203.067281:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776203.067285:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776203.067289:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776203.067291:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776203.067293:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776203.067295:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776203.195296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776203.195304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776203.195311:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776203.195313:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776203.195317:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776203.195318:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776203.195399:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776203.195403:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776203.195406:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776204.091254:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776204.091260:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776204.091263:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776204.091269:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776204.091273:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776204.091276:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776204.091282:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776204.091285:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776204.091288:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776204.219216:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776204.219222:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776204.219226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776204.219227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776204.219235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776204.219236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776204.219324:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776204.219330:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776204.219334:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776204.923248:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307759878, at 4307759936 00000100:00000010:1.0:1775776204.923263:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2f60. 00000100:00000040:0.0:1775776204.923265:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307759878, at 4307759936 00000100:00000010:0.0:1775776204.923280:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2120. 00000100:00000001:1.0:1775776205.115208:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776205.115213:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776205.115214:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776205.115217:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776205.115220:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776205.115222:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776205.115224:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776205.115226:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776205.115227:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776205.243258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776205.243262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776205.243269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776205.243270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776205.243272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776205.243273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776205.243350:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776205.243354:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776205.243357:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776206.139270:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776206.139276:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776206.139278:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776206.139284:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776206.139287:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776206.139291:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776206.139293:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776206.139295:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776206.139298:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776206.139396:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776206.139401:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776206.139404:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776206.139407:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776206.139411:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776206.139418:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13088 00000400:00000200:3.0:1775776206.139427:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776206.139433:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776206.139436:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776206.139439:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776206.139442:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776206.139444:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776206.139446:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776206.139449:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776206.139452:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776206.139454:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776206.139457:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776206.139459:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776206.139461:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776206.139464:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776206.139466:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776206.139467:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776206.139469:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776206.139471:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776206.139475:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776206.139477:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776206.139480:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776206.139483:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776206.139486:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776206.139488:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776206.139490:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776206.139499:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775776206.139502:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776206.139506:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776206.139508:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776206.139510:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776206.139512:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776206.139516:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776206.139523:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000100:00000001:3.0:1775776206.139529:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776206.139537:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710396928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776206.139547:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776206.139551:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776206.139570:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710396928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.139625:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776206.139628:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776206.139630:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776206.139636:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71500 x1862022710396928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.139643:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776206.139644:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776206.139647:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776206.139649:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776206.139652:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776206.139655:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710396928:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776206.139658:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776206.139662:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776206.139664:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776206.139666:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776206.139671:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776206.139673:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776206.139678:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776206.139682:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776206.139689:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776206.139695:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710396928, portal 25 00000100:00000001:1.0:1775776206.139698:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776206.139699:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849009408 : -123823860542208 : ffff8f6202c71500) 00000100:00000040:1.0:1775776206.139704:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71500 x1862022710396928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.139712:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776206.139714:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776206.139717:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776206.139720:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710396928, offset 0 00000400:00000200:1.0:1775776206.139726:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776206.139734:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776206.139740:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776206.139744:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776206.139748:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776206.139751:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776206.139754:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776206.139757:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453653:453653:256:4294967295] 192.168.201.115@tcp LPNI seq info [453578:453578:8:4294967295] 00000400:00000200:1.0:1775776206.139761:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776206.139770:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776206.139775:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776206.139781:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776206.139787:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776206.139791:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775776206.139796:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776206.139800:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1775776206.139804:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776206.139807:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776206.139812:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776206.139817:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776206.139821:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776206.139823:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776206.139829:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776206.139833:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71500 x1862022710396928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776206.139859:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776206.139863:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776206.139866:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776206.139867:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776206.139868:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776206.139871:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776206.139873:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13088 00000400:00000200:3.0:1775776206.139877:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776206.139881:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776206.139883:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776206.139886:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776206.139887:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776206.139889:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776206.139891:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776206.139893:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776206.139894:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776206.139896:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776206.139897:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776206.139899:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776206.139901:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776206.139904:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776206.139906:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776206.139907:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776206.139909:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776206.139910:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776206.139912:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776206.139914:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776206.139917:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776206.139921:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776206.139923:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776206.139925:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776206.139929:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776206.139932:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776206.139934:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776206.139937:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775776206.139940:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776206.139942:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776206.139943:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776206.139945:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776206.139946:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776206.139948:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776206.139952:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000100:00000001:3.0:1775776206.139956:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776206.139960:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710397056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776206.139968:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776206.139970:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776206.139974:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710397056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140004:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776206.140008:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71500 x1862022710396928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140015:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776206.140017:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776206.140023:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776206.140027:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776206.140030:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307764998 00000100:00000010:1.0:1775776206.140035:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2f60. 00000100:00000001:1.0:1775776206.140041:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776206.140047:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307764998 00000100:00000010:1.0:1775776206.140050:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2a80. 00000100:00000001:1.0:1775776206.140052:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776206.140055:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776206.140058:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776206.140060:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776206.140064:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776206/real 1775776206] req@ffff8f6202c71500 x1862022710396928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776206.140073:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776206.140074:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776206.140076:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776206.140079:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776206.140080:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776206.140083:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776206.140087:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71500 x1862022710396928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776206.140095:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71500 x1862022710396928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140102:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776206.140103:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776206.140105:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776206.140107:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776206.140110:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71500 x1862022710396928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140117:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776206.140121:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71500 x1862022710396928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140128:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776206.140131:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776206.140132:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776206.140149:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776206.140152:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776206.140154:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776206.140157:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776206.140161:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776206.140163:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776206.140167:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71500 x1862022710396928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776206.140175:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710396928:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776206.140179:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776206.140181:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776206.140184:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71500 x1862022710396928/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140196:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776206.140198:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776206.140200:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776206.140204:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776206.140206:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776206.140207:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776206.140209:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776206.140211:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36000. 02000000:00000001:1.0:1775776206.140214:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776206.140216:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776206.140217:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71500. 00000100:00000001:1.0:1775776206.140220:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776206.140221:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776206.140223:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776206.140230:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776206.140231:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776206.140233:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776206.140235:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776206.140236:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776206.140240:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70380 x1862022710397056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140247:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776206.140248:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776206.140250:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776206.140252:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776206.140254:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776206.140257:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710397056:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776206.140260:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776206.140262:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776206.140263:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776206.140265:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776206.140268:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776206.140270:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776206.140274:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776206.140276:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776206.140280:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776206.140283:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710397056, portal 10 00000100:00000001:1.0:1775776206.140284:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776206.140286:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004928 : -123823860546688 : ffff8f6202c70380) 00000100:00000040:1.0:1775776206.140291:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70380 x1862022710397056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140297:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776206.140299:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776206.140301:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776206.140303:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710397056, offset 0 00000400:00000200:1.0:1775776206.140307:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776206.140311:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776206.140314:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776206.140317:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776206.140319:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776206.140321:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776206.140323:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776206.140326:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453654:453654:256:4294967295] 192.168.201.115@tcp LPNI seq info [453579:453579:8:4294967295] 00000400:00000200:1.0:1775776206.140329:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776206.140335:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776206.140339:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776206.140342:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776206.140346:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776206.140349:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775776206.140353:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776206.140355:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1775776206.140359:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776206.140361:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776206.140363:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776206.140367:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776206.140369:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776206.140371:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776206.140374:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776206.140377:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70380 x1862022710397056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140385:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776206.140387:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70380 x1862022710397056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140394:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776206.140395:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776206.140398:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776206.140402:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776206.140405:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776206.140407:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307761163 00000100:00000001:1.0:1775776206.140410:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307764998 00000100:00000001:1.0:1775776206.140412:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307761163 00000100:00000001:1.0:1775776206.140413:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307764998 00000100:00000001:1.0:1775776206.140415:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776206.140417:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776206.140419:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776206.140422:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776206/real 1775776206] req@ffff8f6202c70380 x1862022710397056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776206.140429:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776206.140430:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776206.140432:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776206.140434:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776206.140436:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776206.140438:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776206.140441:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70380 x1862022710397056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776206.140448:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70380 x1862022710397056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140459:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776206.140461:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776206.140462:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776206.140464:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776206.140467:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70380 x1862022710397056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140473:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776206.140476:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70380 x1862022710397056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140482:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776206.140484:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776206.140485:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776206.140487:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776206.140488:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776206.140493:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776206.140495:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776206.140498:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776206.140501:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776206.140504:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70380 x1862022710397056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776206.140511:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710397056:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776206.140514:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776206.140516:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776206.140518:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70380 x1862022710397056/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140524:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776206.140527:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776206.140529:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776206.140531:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776206.140533:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776206.140534:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776206.140536:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776206.140538:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36400. 02000000:00000001:1.0:1775776206.140540:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776206.140541:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776206.140543:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70380. 00000100:00000001:1.0:1775776206.140545:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776206.140546:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776206.140547:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776206.140551:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776206.140553:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776206.140554:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776206.140566:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776206.140622:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776206.140626:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776206.140630:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776206.140631:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776206.140633:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1775776206.140633:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776206.140635:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776206.140638:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776206.140642:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13088 00000400:00000200:3.0:1775776206.140647:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776206.140650:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776206.140652:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776206.140655:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776206.140657:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776206.140659:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776206.140661:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776206.140663:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776206.140665:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776206.140666:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776206.140668:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776206.140670:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776206.140672:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776206.140675:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776206.140677:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776206.140678:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776206.140681:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776206.140683:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776206.140685:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776206.140687:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776206.140689:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776206.140692:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776206.140694:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776206.140696:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776206.140698:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776206.140700:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776206.140702:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776206.140705:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775776206.140708:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776206.140710:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776206.140712:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776206.140713:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776206.140714:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776206.140716:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776206.140720:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775776206.140724:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776206.140728:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710397184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776206.140734:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776206.140737:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776206.140740:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710397184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776206.140754:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776206.140757:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776206.140761:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776206.140764:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776206.140767:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776206.140768:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776206.140771:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776206.140773:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776206.140774:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13098) 00000100:00000040:1.0:1775776206.140779:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73480 x1862022710397184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140785:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776206.140787:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776206.140789:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776206.140791:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776206.140792:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776206.140795:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710397184:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776206.140798:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776206.140799:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776206.140800:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776206.140802:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776206.140804:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776206.140806:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776206.140809:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776206.140810:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776206.140812:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776206.140815:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710397184, portal 10 00000100:00000001:1.0:1775776206.140816:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776206.140817:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849017472 : -123823860534144 : ffff8f6202c73480) 00000100:00000040:1.0:1775776206.140821:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73480 x1862022710397184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140825:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776206.140827:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776206.140828:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776206.140829:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710397184, offset 0 00000400:00000200:1.0:1775776206.140832:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776206.140835:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776206.140837:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776206.140839:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776206.140840:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776206.140842:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776206.140844:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776206.140846:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453655:453655:256:4294967295] 192.168.201.115@tcp LPNI seq info [453580:453580:8:4294967295] 00000400:00000200:1.0:1775776206.140848:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776206.140852:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776206.140855:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776206.140857:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776206.140859:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776206.140861:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775776206.140864:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776206.140866:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1775776206.140868:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776206.140869:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776206.140871:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776206.140873:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776206.140875:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776206.140877:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776206.140879:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776206.140881:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73480 x1862022710397184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140886:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776206.140888:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73480 x1862022710397184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140892:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776206.140893:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776206.140895:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776206.140897:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776206.140899:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776206.140900:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307761164 00000100:00000001:1.0:1775776206.140901:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307764998 00000100:00000001:1.0:1775776206.140903:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307761164 00000100:00000001:1.0:1775776206.140904:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307764998 00000100:00000001:1.0:1775776206.140906:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776206.140907:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776206.140908:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776206.140910:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776206/real 1775776206] req@ffff8f6202c73480 x1862022710397184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776206.140915:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776206.140915:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776206.140916:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776206.140917:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776206.140918:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776206.140919:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776206.140921:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73480 x1862022710397184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776206.140926:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73480 x1862022710397184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140930:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776206.140931:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776206.140932:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776206.140933:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776206.140935:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73480 x1862022710397184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140939:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776206.140941:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73480 x1862022710397184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140944:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776206.140946:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776206.140946:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776206.140947:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776206.140948:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776206.140949:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776206.140951:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776206.140952:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776206.140953:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776206.140955:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73480 x1862022710397184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776206.140960:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710397184:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776206.140962:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776206.140963:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776206.140964:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73480 x1862022710397184/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776206.140968:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776206.140969:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776206.140970:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776206.140972:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776206.140973:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776206.140974:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776206.140975:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776206.140976:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34c00. 02000000:00000001:1.0:1775776206.140977:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776206.140978:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776206.140979:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73480. 00000100:00000001:1.0:1775776206.140981:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776206.140982:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776206.140983:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776206.140985:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776206.140986:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776206.140987:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776206.140988:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776206.267250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776206.267258:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776206.267259:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776206.267261:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776206.267265:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776206.267267:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776206.267268:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776206.267270:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776206.267272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776206.267273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776206.267275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776206.267276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776206.267340:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776206.267344:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776206.267346:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776207.163238:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776207.163247:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776207.163249:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776207.163256:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776207.163260:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776207.163265:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776207.163268:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776207.163271:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776207.163274:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776207.291281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776207.291289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776207.291296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776207.291297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776207.291301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776207.291302:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776207.291365:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776207.291369:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776207.291373:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776208.187258:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776208.187265:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776208.187267:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776208.187272:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776208.187276:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776208.187279:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776208.187282:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776208.187284:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776208.187286:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776208.315242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776208.315247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776208.315251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776208.315252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776208.315256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776208.315257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776208.315317:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776208.315321:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776208.315324:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776209.211249:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776209.211255:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776209.211256:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776209.211262:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776209.211265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776209.211269:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776209.211272:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776209.211274:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776209.211277:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776209.339298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776209.339305:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776209.339313:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776209.339317:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776209.339319:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776209.339323:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776209.339327:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776209.339329:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776209.339331:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776209.339334:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776209.339338:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776209.339339:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776209.339340:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776209.339342:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776209.339343:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776209.339345:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776209.339346:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776209.339348:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776209.339411:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776209.339415:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776209.339417:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776210.043260:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307764998, at 4307765056 00000100:00000040:0.0:1775776210.043267:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307764998, at 4307765056 00000100:00000010:0.0:1775776210.043282:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2f60. 00000100:00000010:1.0:1775776210.043286:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2a80. 00000100:00000001:1.0:1775776210.235267:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776210.235275:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776210.235278:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776210.235285:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776210.235289:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776210.235294:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776210.235297:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776210.235299:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776210.235302:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776210.363282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776210.363290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776210.363296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776210.363297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776210.363301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776210.363303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776210.363368:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776210.363372:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776210.363376:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776211.259234:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776211.259235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776211.259240:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776211.259242:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776211.259242:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1775776211.259246:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776211.259247:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776211.259251:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776211.259252:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776211.259254:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776211.259256:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776211.259257:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776211.259260:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776211.259261:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776211.259265:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13093 00000400:00000200:3.0:1775776211.259274:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776211.259281:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776211.259284:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776211.259288:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776211.259291:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776211.259293:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776211.259296:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776211.259299:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776211.259301:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776211.259304:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776211.259307:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776211.259309:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776211.259311:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776211.259314:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776211.259316:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776211.259318:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776211.259320:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776211.259323:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776211.259326:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776211.259329:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776211.259333:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776211.259336:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776211.259339:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776211.259341:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776211.259343:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776211.259354:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775776211.259357:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776211.259361:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776211.259363:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776211.259366:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776211.259368:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776211.259372:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776211.259379:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775776211.259386:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776211.259395:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710397312/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776211.259405:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776211.259409:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776211.259413:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710397312/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776211.259434:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776211.259437:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776211.259440:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776211.259442:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776211.259444:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776211.259447:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776211.259450:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13093 00000100:00000001:1.0:1775776211.259451:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776211.259454:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1775776211.259455:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776211.259456:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000400:00000200:3.0:1775776211.259461:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000040:1.0:1775776211.259464:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71c00 x1862022710397312/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775776211.259465:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776211.259480:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776211.259482:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776211.259484:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776211.259486:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776211.259488:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776211.259491:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776211.259494:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776211.259498:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776211.259502:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776211.259507:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1775776211.259508:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776211.259510:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776211.259513:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1775776211.259513:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776211.259516:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776211.259517:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775776211.259519:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776211.259522:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00100000:1.0:1775776211.259525:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710397312:192.168.201.115@tcp:250:kworker.0 00000100:00000001:3.0:1775776211.259526:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:1.0:1775776211.259529:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1775776211.259531:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:1.0:1775776211.259533:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776211.259536:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776211.259536:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000001:1.0:1775776211.259540:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775776211.259540:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776211.259543:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:1.0:1775776211.259544:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776211.259547:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776211.259548:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775776211.259553:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776211.259553:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776211.259556:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000100:00000001:3.0:1775776211.259556:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776211.259560:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000010:1.0:1775776211.259562:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000008:00000001:3.0:1775776211.259563:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776211.259566:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776211.259568:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710397312, portal 25 00000100:00000001:1.0:1775776211.259571:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000010:3.0:1775776211.259571:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000100:00000001:1.0:1775776211.259572:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849011200 : -123823860540416 : ffff8f6202c71c00) 00000020:00000040:3.0:1775776211.259575:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000040:1.0:1775776211.259579:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71c00 x1862022710397312/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775776211.259580:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776211.259583:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776211.259587:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776211.259590:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775776211.259591:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1775776211.259593:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776211.259594:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776211.259598:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776211.259600:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710397312, offset 0 02000000:00000010:3.0:1775776211.259600:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000100:00000001:3.0:1775776211.259606:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776211.259610:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00100000:3.0:1775776211.259615:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710397440/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776211.259618:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776211.259626:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:3.0:1775776211.259629:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775776211.259632:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1775776211.259633:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776211.259636:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000040:3.0:1775776211.259639:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710397440/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776211.259641:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776211.259645:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776211.259652:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453656:453656:256:4294967295] 192.168.201.115@tcp LPNI seq info [453581:453581:8:4294967295] 00000400:00000200:1.0:1775776211.259661:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776211.259662:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776211.259666:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776211.259672:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00080000:3.0:1775776211.259675:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000200:1.0:1775776211.259679:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776211.259679:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776211.259683:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000010:1.0:1775776211.259685:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000100:00000001:3.0:1775776211.259686:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776211.259691:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13093 00000800:00000200:1.0:1775776211.259692:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776211.259697:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000400:00000200:3.0:1775776211.259699:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000200:1.1:1775776211.259703:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000400:00000200:3.0:1775776211.259706:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000800:00000200:1.1:1775776211.259707:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000100:00080000:3.0:1775776211.259711:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775776211.259714:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776211.259717:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000100:00000040:3.0:1775776211.259718:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776211.259721:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776211.259723:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:3.0:1775776211.259725:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776211.259729:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1775776211.259730:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776211.259733:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000020:00000001:3.0:1775776211.259733:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000010:1.0:1775776211.259737:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000020:00000040:3.0:1775776211.259737:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776211.259741:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776211.259743:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000100:00000001:1.0:1775776211.259744:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:3.0:1775776211.259747:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000200:1.0:1775776211.259749:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71c00 x1862022710397312/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775776211.259750:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776211.259758:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776211.259760:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1775776211.259761:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776211.259765:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71c00 x1862022710397312/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776211.259766:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776211.259771:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776211.259776:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00000001:1.0:1775776211.259777:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776211.259780:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1775776211.259781:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776211.259784:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776211.259785:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776211.259788:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776211.259789:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776211.259792:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307770118 02000000:00000001:3.0:1775776211.259793:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:1.0:1775776211.259795:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2a80. 02000000:00000001:3.0:1775776211.259796:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776211.259799:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776211.259802:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000008:00100000:3.0:1775776211.259802:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776211.259806:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776211.259808:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776211.259809:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307770118 00000100:00000010:1.0:1775776211.259815:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2720. 00000100:00000010:3.0:1775776211.259815:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000100:00000001:1.0:1775776211.259818:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000020:00000040:3.0:1775776211.259819:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000100:00000001:1.0:1775776211.259821:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776211.259823:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776211.259823:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776211.259825:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776211.259826:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1775776211.259830:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:1.0:1775776211.259831:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776211/real 1775776211] req@ffff8f6202c71c00 x1862022710397312/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775776211.259833:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776211.259837:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1775776211.259841:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776211.259843:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000010:3.0:1775776211.259843:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000400:00000010:1.0:1775776211.259844:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776211.259847:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776211.259849:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:3.0:1775776211.259850:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776211.259851:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776211.259856:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71c00 x1862022710397312/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776211.259859:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710397568/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1775776211.259870:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71c00 x1862022710397312/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776211.259878:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776211.259879:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776211.259881:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776211.259882:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776211.259883:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776211.259885:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776211.259890:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71c00 x1862022710397312/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776211.259890:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710397568/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776211.259907:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776211.259913:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71c00 x1862022710397312/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776211.259914:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776211.259919:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776211.259923:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776211.259925:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776211.259925:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776211.259929:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776211.259931:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776211.259931:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776211.259933:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776211.259935:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:3.0:1775776211.259938:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00080000:1.0:1775776211.259939:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776211.259945:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000040:3.0:1775776211.259947:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13103) 00000100:00000001:1.0:1775776211.259948:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776211.259953:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71c00 x1862022710397312/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776211.259958:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710397312:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776211.259960:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776211.259961:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776211.259963:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71c00 x1862022710397312/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776211.259967:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776211.259969:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776211.259970:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776211.259972:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776211.259973:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776211.259974:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776211.259975:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776211.259976:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34c00. 02000000:00000001:1.0:1775776211.259978:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776211.259979:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776211.259981:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71c00. 00000100:00000001:1.0:1775776211.259982:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776211.259983:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776211.259984:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776211.259989:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776211.259990:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776211.259991:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776211.259992:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776211.259993:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776211.259995:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70e00 x1862022710397440/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776211.259999:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776211.260000:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776211.260001:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776211.260002:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776211.260004:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776211.260005:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710397440:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776211.260007:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776211.260008:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776211.260009:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776211.260010:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776211.260011:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776211.260012:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776211.260015:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776211.260016:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776211.260018:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776211.260019:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710397440, portal 10 00000100:00000001:1.0:1775776211.260020:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776211.260021:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849007616 : -123823860544000 : ffff8f6202c70e00) 00000100:00000040:1.0:1775776211.260025:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70e00 x1862022710397440/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776211.260029:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776211.260030:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776211.260031:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776211.260033:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710397440, offset 0 00000400:00000200:1.0:1775776211.260035:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776211.260037:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776211.260039:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776211.260040:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776211.260042:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776211.260043:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776211.260045:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776211.260047:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453657:453657:256:4294967295] 192.168.201.115@tcp LPNI seq info [453582:453582:8:4294967295] 00000400:00000200:1.0:1775776211.260049:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776211.260060:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776211.260062:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776211.260065:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776211.260068:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776211.260069:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775776211.260072:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776211.260073:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1775776211.260075:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776211.260076:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776211.260078:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776211.260080:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776211.260082:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776211.260083:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776211.260084:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776211.260086:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70e00 x1862022710397440/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776211.260091:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776211.260092:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70e00 x1862022710397440/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776211.260096:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776211.260097:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776211.260099:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776211.260101:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776211.260103:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776211.260104:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307766283 00000100:00000001:1.0:1775776211.260106:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307770118 00000100:00000001:1.0:1775776211.260107:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307766283 00000100:00000001:1.0:1775776211.260108:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307770118 00000100:00000001:1.0:1775776211.260110:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776211.260111:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776211.260112:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776211.260113:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776211/real 1775776211] req@ffff8f6202c70e00 x1862022710397440/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776211.260121:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776211.260122:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776211.260123:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776211.260124:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776211.260125:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776211.260126:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776211.260128:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70e00 x1862022710397440/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776211.260132:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70e00 x1862022710397440/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776211.260150:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776211.260151:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776211.260152:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776211.260153:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776211.260155:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70e00 x1862022710397440/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776211.260159:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776211.260161:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70e00 x1862022710397440/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776211.260165:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776211.260166:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776211.260167:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776211.260168:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776211.260169:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776211.260172:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776211.260173:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776211.260175:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776211.260176:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776211.260178:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70e00 x1862022710397440/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776211.260183:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710397440:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776211.260185:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776211.260187:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70000 x1862022710397568/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776211.260190:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776211.260191:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776211.260192:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776211.260193:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776211.260194:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776211.260195:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710397568:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776211.260197:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776211.260198:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776211.260199:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776211.260200:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776211.260202:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776211.260203:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776211.260205:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776211.260206:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776211.260208:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776211.260210:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710397568, portal 10 00000100:00000001:1.0:1775776211.260211:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776211.260212:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004032 : -123823860547584 : ffff8f6202c70000) 00000100:00000040:1.0:1775776211.260215:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70000 x1862022710397568/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776211.260218:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776211.260219:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776211.260221:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776211.260222:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710397568, offset 0 00000400:00000200:1.0:1775776211.260225:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776211.260226:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776211.260228:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776211.260230:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776211.260232:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776211.260233:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776211.260234:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776211.260236:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453658:453658:256:4294967295] 192.168.201.115@tcp LPNI seq info [453583:453583:8:4294967295] 00000400:00000200:1.0:1775776211.260239:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776211.260242:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776211.260244:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776211.260246:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776211.260248:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776211.260249:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775776211.260252:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776211.260253:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1775776211.260255:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776211.260256:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776211.260258:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776211.260260:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776211.260261:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776211.260262:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776211.260264:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776211.260265:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70000 x1862022710397568/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776211.260270:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776211.260271:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70000 x1862022710397568/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776211.260275:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776211.260276:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776211.260277:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776211.260279:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776211.260280:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776211.260282:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307766284 00000100:00000001:1.0:1775776211.260283:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307770118 00000100:00000001:1.0:1775776211.260284:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307766284 00000100:00000001:1.0:1775776211.260286:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307770118 00000100:00000001:1.0:1775776211.260286:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776211.260288:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776211.260289:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776211.260290:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776211/real 1775776211] req@ffff8f6202c70000 x1862022710397568/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776211.260294:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776211.260295:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776211.260296:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776211.260297:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776211.260298:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776211.260299:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776211.260300:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70000 x1862022710397568/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776211.260305:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70000 x1862022710397568/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776211.260308:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776211.260309:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776211.260310:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776211.260310:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776211.260312:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70000 x1862022710397568/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776211.260315:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776211.260317:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70000 x1862022710397568/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776211.260321:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776211.260323:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776211.260323:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776211.260324:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776211.260325:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776211.260326:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776211.260328:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776211.260329:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776211.260331:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776211.260332:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70000 x1862022710397568/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776211.260336:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710397568:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776211.260338:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776211.260339:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776211.260340:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70e00 x1862022710397440/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776211.260343:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776211.260344:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776211.260345:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776211.260346:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776211.260347:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776211.260348:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776211.260349:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776211.260350:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36400. 02000000:00000001:1.0:1775776211.260351:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776211.260351:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776211.260352:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70e00. 00000100:00000001:1.0:1775776211.260354:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776211.260355:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776211.260356:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776211.260357:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70000 x1862022710397568/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776211.260369:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776211.260370:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776211.260370:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776211.260372:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776211.260372:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776211.260373:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776211.260374:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776211.260375:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36000. 02000000:00000001:1.0:1775776211.260376:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776211.260376:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776211.260377:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70000. 00000100:00000001:1.0:1775776211.260378:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776211.260378:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776211.260379:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776211.260381:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776211.260382:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776211.260383:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776211.260384:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776211.387227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776211.387233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776211.387237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776211.387238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776211.387242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776211.387243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776211.387319:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776211.387323:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776211.387326:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776212.283225:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776212.283230:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776212.283231:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776212.283235:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776212.283238:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776212.283240:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776212.283242:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776212.283244:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776212.283246:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776212.411230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776212.411236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776212.411241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776212.411243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776212.411247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776212.411248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776212.411317:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776212.411320:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776212.411323:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776213.307228:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776213.307242:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776213.307244:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776213.307250:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776213.307254:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776213.307258:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776213.307259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776213.307261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776213.307263:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776213.435235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776213.435239:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776213.435243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776213.435243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776213.435246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776213.435246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776213.435304:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776213.435308:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776213.435311:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776214.331208:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776214.331213:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776214.331215:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776214.331219:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776214.331221:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776214.331224:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776214.331226:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776214.331228:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776214.331230:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776214.459271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776214.459278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776214.459285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776214.459286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776214.459290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776214.459291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776214.459352:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776214.459356:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776214.459358:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776215.163218:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307770118, at 4307770176 00000100:00000010:0.0:1775776215.163231:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2a80. 00000100:00000040:1.0:1775776215.163247:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307770118, at 4307770176 00000100:00000010:1.0:1775776215.163261:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2720. 00000100:00000001:1.0:1775776215.355266:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776215.355273:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776215.355275:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776215.355281:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776215.355285:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776215.355289:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776215.355291:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776215.355294:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776215.355297:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776215.483248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776215.483256:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776215.483258:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776215.483261:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776215.483266:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776215.483268:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776215.483269:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776215.483271:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776215.483274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776215.483275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776215.483278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776215.483279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776215.483353:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776215.483358:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776215.483362:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776216.379206:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776216.379213:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776216.379215:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776216.379220:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776216.379224:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776216.379228:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775776216.379231:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776216.379231:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776216.379234:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776216.379239:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776216.379240:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776216.379244:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776216.379249:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776216.379254:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776216.379262:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13098 00000400:00000200:0.0:1775776216.379274:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776216.379282:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776216.379286:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776216.379290:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776216.379293:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776216.379295:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776216.379296:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776216.379300:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776216.379302:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776216.379304:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776216.379308:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776216.379309:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776216.379311:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776216.379313:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776216.379314:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776216.379315:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776216.379316:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776216.379318:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776216.379320:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776216.379321:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776216.379324:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776216.379327:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776216.379329:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776216.379330:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776216.379332:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776216.379342:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062d80. 00000020:00000040:0.0:1775776216.379344:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776216.379347:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776216.379355:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776216.379358:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776216.379360:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776216.379364:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776216.379372:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6c00. 00000100:00000001:0.0:1775776216.379389:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776216.379399:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062d80 x1862022710397696/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776216.379407:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776216.379412:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776216.379415:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009965f80b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062d80 x1862022710397696/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776216.379430:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776216.379432:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776216.379436:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776216.379437:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776216.379438:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776216.379440:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776216.379442:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13098 00000400:00000200:0.0:1775776216.379445:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776216.379448:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776216.379449:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776216.379452:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776216.379453:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776216.379454:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776216.379457:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776216.379458:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776216.379459:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776216.379461:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776216.379462:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:1.0:1775776216.379462:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1775776216.379464:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776216.379466:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775776216.379469:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:1.0:1775776216.379470:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e062d80 x1862022710397696/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775776216.379471:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775776216.379477:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776216.379481:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776216.379483:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776216.379484:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776216.379485:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775776216.379486:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776216.379487:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776216.379491:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776216.379493:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776216.379495:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776216.379497:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776216.379498:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776216.379501:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776216.379502:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776216.379503:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776216.379506:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1775776216.379509:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775776216.379510:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e062d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710397696:192.168.201.115@tcp:250:kworker.0 00000008:00000001:0.0:1775776216.379515:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776216.379516:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1775776216.379517:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776216.379520:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062a00. 02000000:00000001:1.0:1775776216.379521:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:0.0:1775776216.379522:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776216.379524:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776216.379525:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776216.379527:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776216.379527:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1775776216.379530:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776216.379531:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776216.379533:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776216.379534:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d000. 02000000:00000010:0.0:1775776216.379536:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:1.0:1775776216.379537:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776216.379540:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776216.379545:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00100000:0.0:1775776216.379546:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062a00 x1862022710397824/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1775776216.379549:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000100:00000001:0.0:1775776216.379554:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776216.379556:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776216.379558:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009753f9f6] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062a00 x1862022710397824/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1775776216.379562:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000001:0.0:1775776216.379563:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776216.379566:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776216.379569:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776216.379570:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776216.379571:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1775776216.379572:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710397696, portal 25 00000100:00000001:0.0:1775776216.379574:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776216.379575:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1775776216.379577:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13098 00000100:00000001:1.0:1775776216.379578:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574564736 : -123823134986880 : ffff8f622e062d80) 00000400:00000200:0.0:1775776216.379581:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776216.379584:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776216.379585:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776216.379587:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e062d80 x1862022710397696/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776216.379589:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776216.379593:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776216.379596:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776216.379600:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776216.379603:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776216.379605:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000100:00000001:1.0:1775776216.379606:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1775776216.379607:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776216.379609:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775776216.379613:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:1.0:1775776216.379613:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000020:00000001:0.0:1775776216.379616:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000010:1.0:1775776216.379616:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000040:0.0:1775776216.379621:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000200:1.0:1775776216.379621:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710397696, offset 0 00000100:00000001:0.0:1775776216.379625:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776216.379626:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776216.379627:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776216.379628:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776216.379631:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00080000:0.0:1775776216.379633:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776216.379635:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776216.379636:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776216.379638:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776216.379639:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776216.379640:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776216.379642:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1775776216.379642:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000008:00000001:0.0:1775776216.379646:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776216.379648:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776216.379650:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061500. 00000020:00000040:0.0:1775776216.379653:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:1.0:1775776216.379653:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 02000000:00000001:0.0:1775776216.379656:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776216.379657:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776216.379658:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776216.379659:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776216.379661:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776216.379661:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000010:0.0:1775776216.379664:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7000. 00000400:00000200:1.0:1775776216.379666:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:0.0:1775776216.379667:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776216.379673:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061500 x1862022710397952/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776216.379674:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776216.379680:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776216.379687:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453659:453659:256:4294967295] 192.168.201.115@tcp LPNI seq info [453584:453584:8:4294967295] 00000100:00000001:0.0:1775776216.379696:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776216.379698:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776216.379700:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:0.0:1775776216.379703:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb8d992] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061500 x1862022710397952/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776216.379709:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776216.379711:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776216.379716:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776216.379718:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776216.379721:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000040:0.0:1775776216.379722:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776216.379728:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13109) 00000800:00000200:1.0:1775776216.379734:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776216.379742:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776216.379750:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776216.379754:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775776216.379760:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776216.379764:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775776216.379768:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776216.379771:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776216.379779:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776216.379784:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776216.379790:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776216.379792:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776216.379799:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776216.379802:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e062d80 x1862022710397696/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.379809:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776216.379811:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e062d80 x1862022710397696/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776241 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.379816:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776216.379817:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776216.379823:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776216.379826:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776216.379829:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307775238 00000100:00000010:1.0:1775776216.379833:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2720. 00000100:00000001:1.0:1775776216.379840:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776216.379851:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307775238 00000100:00000010:1.0:1775776216.379854:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2600. 00000100:00000001:1.0:1775776216.379856:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776216.379858:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776216.379860:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776216.379862:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776216.379864:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776216/real 1775776216] req@ffff8f622e062d80 x1862022710397696/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776216.379871:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776216.379872:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776216.379873:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776216.379875:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776216.379876:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776216.379878:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776216.379880:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e062d80 x1862022710397696/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776216.379885:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e062d80 x1862022710397696/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.379889:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776216.379890:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776216.379892:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776216.379893:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776216.379895:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e062d80 x1862022710397696/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.379899:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776216.379902:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e062d80 x1862022710397696/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.379906:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776216.379908:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776216.379908:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776216.379909:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776216.379911:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776216.379912:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776216.379914:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776216.379917:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776216.379919:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776216.379922:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e062d80 x1862022710397696/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776216.379927:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e062d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710397696:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776216.379930:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776216.379931:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776216.379932:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e062d80 x1862022710397696/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.379936:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776216.379938:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776216.379940:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776216.379942:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776216.379943:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776216.379944:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776216.379946:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776216.379947:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e6c00. 02000000:00000001:1.0:1775776216.379950:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776216.379951:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776216.379952:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e062d80. 00000100:00000001:1.0:1775776216.379954:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776216.379955:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776216.379956:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776216.379963:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776216.379964:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776216.379966:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776216.379967:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776216.379968:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776216.379969:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e062a00 x1862022710397824/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.379974:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776216.379975:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776216.379977:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776216.379978:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776216.379980:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776216.379981:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e062a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710397824:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776216.379983:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776216.379985:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776216.379985:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776216.379986:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776216.379988:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776216.379989:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776216.379992:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776216.379994:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776216.379996:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776216.379998:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710397824, portal 10 00000100:00000001:1.0:1775776216.379999:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776216.380000:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574563840 : -123823134987776 : ffff8f622e062a00) 00000100:00000040:1.0:1775776216.380005:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e062a00 x1862022710397824/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.380009:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776216.380011:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776216.380012:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776216.380013:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710397824, offset 0 00000400:00000200:1.0:1775776216.380016:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776216.380018:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776216.380020:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776216.380022:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776216.380024:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776216.380025:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776216.380027:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776216.380029:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453660:453660:256:4294967295] 192.168.201.115@tcp LPNI seq info [453585:453585:8:4294967295] 00000400:00000200:1.0:1775776216.380032:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776216.380036:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776216.380039:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776216.380041:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776216.380043:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776216.380045:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775776216.380048:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776216.380050:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775776216.380052:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776216.380054:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776216.380055:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776216.380058:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776216.380059:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776216.380060:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776216.380062:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776216.380064:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e062a00 x1862022710397824/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.380069:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776216.380070:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e062a00 x1862022710397824/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776241 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.380074:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776216.380074:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776216.380076:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776216.380078:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776216.380080:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776216.380081:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307771403 00000100:00000001:1.0:1775776216.380083:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307775238 00000100:00000001:1.0:1775776216.380084:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307771403 00000100:00000001:1.0:1775776216.380085:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307775238 00000100:00000001:1.0:1775776216.380087:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776216.380089:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776216.380090:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776216.380092:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776216/real 1775776216] req@ffff8f622e062a00 x1862022710397824/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776216.380097:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776216.380098:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776216.380099:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776216.380100:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776216.380101:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776216.380102:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776216.380104:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e062a00 x1862022710397824/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776216.380108:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e062a00 x1862022710397824/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.380111:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776216.380112:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776216.380112:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776216.380113:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776216.380115:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e062a00 x1862022710397824/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.380118:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776216.380120:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e062a00 x1862022710397824/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.380123:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776216.380125:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776216.380126:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776216.380127:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776216.380128:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776216.380132:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776216.380151:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776216.380153:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776216.380155:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776216.380158:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e062a00 x1862022710397824/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776216.380163:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e062a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710397824:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776216.380165:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776216.380167:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061500 x1862022710397952/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.380170:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776216.380171:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776216.380173:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776216.380174:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776216.380175:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776216.380177:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710397952:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776216.380178:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776216.380179:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776216.380180:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776216.380181:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776216.380183:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776216.380184:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776216.380186:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776216.380188:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776216.380189:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776216.380191:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710397952, portal 10 00000100:00000001:1.0:1775776216.380193:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776216.380194:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574558464 : -123823134993152 : ffff8f622e061500) 00000100:00000040:1.0:1775776216.380197:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061500 x1862022710397952/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.380201:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776216.380203:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776216.380204:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776216.380206:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710397952, offset 0 00000400:00000200:1.0:1775776216.380209:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776216.380211:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776216.380212:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776216.380214:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776216.380216:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776216.380217:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776216.380219:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776216.380221:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453661:453661:256:4294967295] 192.168.201.115@tcp LPNI seq info [453586:453586:8:4294967295] 00000400:00000200:1.0:1775776216.380224:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776216.380233:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776216.380236:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776216.380240:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776216.380243:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776216.380244:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775776216.380247:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776216.380249:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775776216.380251:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776216.380252:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776216.380254:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776216.380256:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776216.380258:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776216.380259:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776216.380261:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776216.380263:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061500 x1862022710397952/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.380267:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776216.380269:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061500 x1862022710397952/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776241 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.380273:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776216.380274:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776216.380275:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776216.380277:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776216.380279:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776216.380280:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307771404 00000100:00000001:1.0:1775776216.380282:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307775238 00000100:00000001:1.0:1775776216.380283:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307771404 00000100:00000001:1.0:1775776216.380285:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307775238 00000100:00000001:1.0:1775776216.380286:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776216.380287:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776216.380288:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776216.380289:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776216/real 1775776216] req@ffff8f622e061500 x1862022710397952/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776216.380294:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776216.380295:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776216.380296:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776216.380297:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776216.380298:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776216.380299:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776216.380301:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061500 x1862022710397952/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776216.380306:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061500 x1862022710397952/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.380309:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776216.380310:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776216.380311:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776216.380312:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776216.380313:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061500 x1862022710397952/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.380317:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776216.380319:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061500 x1862022710397952/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.380322:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776216.380324:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776216.380325:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776216.380326:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776216.380327:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776216.380328:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776216.380330:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776216.380331:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776216.380332:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776216.380334:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061500 x1862022710397952/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776216.380338:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710397952:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776216.380340:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776216.380341:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776216.380342:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e062a00 x1862022710397824/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.380346:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776216.380346:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776216.380347:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d000. 02000000:00000001:1.0:1775776216.380349:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776216.380349:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776216.380350:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776216.380352:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776216.380353:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:1.0:1775776216.380354:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776216.380355:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776216.380356:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e062a00. 00000100:00000001:1.0:1775776216.380357:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776216.380359:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776216.380359:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776216.380361:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061500 x1862022710397952/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776241 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776216.380369:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776216.380370:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776216.380371:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776216.380372:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776216.380373:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776216.380374:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776216.380374:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776216.380375:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e7000. 02000000:00000001:1.0:1775776216.380386:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776216.380387:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776216.380388:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061500. 00000100:00000001:1.0:1775776216.380389:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776216.380390:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776216.380391:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776216.380393:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776216.380393:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776216.380395:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776216.380395:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776216.507309:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776216.507315:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776216.507321:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776216.507322:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776216.507325:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776216.507326:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776216.507368:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776216.507371:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776216.507374:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776217.403244:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776217.403248:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776217.403250:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776217.403255:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776217.403257:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776217.403261:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776217.403263:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776217.403266:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776217.403269:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776217.531240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776217.531248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776217.531255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776217.531256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776217.531260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776217.531261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776217.531335:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776217.531339:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776217.531343:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776218.427290:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776218.427300:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776218.427302:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776218.427310:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776218.427315:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776218.427320:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776218.427323:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776218.427326:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776218.427329:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776218.555275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776218.555281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776218.555287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776218.555288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776218.555292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776218.555294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776218.555382:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776218.555385:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776218.555388:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776219.451315:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776219.451321:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776219.451347:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776219.451352:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776219.451354:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776219.451357:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776219.451359:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776219.451361:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776219.451363:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776219.579251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776219.579258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776219.579264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776219.579268:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776219.579270:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776219.579274:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776219.579277:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776219.579279:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776219.579281:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776219.579283:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776219.579288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776219.579290:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776219.579291:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776219.579293:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776219.579295:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776219.579296:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776219.579298:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776219.579300:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776219.579376:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776219.579381:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776219.579384:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776220.283214:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307775238, at 4307775296 00000100:00000010:1.0:1775776220.283227:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2600. 00000800:00000200:0.0:1775776220.283240:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:0.0:1775776220.283247:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:0.0:1775776220.283252:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000100:00000040:0.0:1775776220.283285:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307775238, at 4307775296 00000100:00000010:0.0:1775776220.283292:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2720. 00000800:00000001:0.0:1775776220.283389:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000200:1.0:1775776220.283398:0:2404:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1775776220.283403:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775776220.283407:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776220.283410:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776220.475246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776220.475252:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776220.475254:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776220.475258:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776220.475261:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776220.475264:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776220.475266:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776220.475268:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776220.475270:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776220.603247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776220.603252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776220.603256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776220.603257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776220.603259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776220.603260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776220.603325:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776220.603329:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776220.603331:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776221.499231:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776221.499232:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776221.499237:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1775776221.499239:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776221.499239:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776221.499242:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1775776221.499244:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:0.0:1775776221.499246:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776221.499248:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776221.499250:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776221.499251:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776221.499254:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775776221.499256:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13104 00000100:00000001:1.0:1775776221.499258:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776221.499261:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776221.499267:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776221.499272:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776221.499276:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776221.499279:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776221.499281:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776221.499284:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776221.499285:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776221.499301:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776221.499304:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776221.499307:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776221.499309:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776221.499311:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776221.499312:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776221.499315:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776221.499316:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776221.499317:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776221.499319:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776221.499321:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776221.499323:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776221.499325:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776221.499327:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776221.499329:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776221.499332:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776221.499333:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776221.499335:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776221.499342:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063b80. 00000020:00000040:0.0:1775776221.499345:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776221.499348:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776221.499349:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776221.499351:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776221.499353:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776221.499356:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776221.499362:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5c00. 00000100:00000001:0.0:1775776221.499367:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776221.499380:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063b80 x1862022710398080/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776221.499388:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776221.499391:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776221.499395:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ee9007b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063b80 x1862022710398080/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776221.499412:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776221.499415:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776221.499419:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776221.499421:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776221.499422:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776221.499425:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776221.499428:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13104 00000400:00000200:0.0:1775776221.499433:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776221.499437:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776221.499440:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776221.499444:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776221.499445:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776221.499447:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776221.499450:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1775776221.499451:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776221.499453:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1775776221.499462:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:1.0:1775776221.499463:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e063b80 x1862022710398080/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775776221.499468:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776221.499471:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776221.499475:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776221.499478:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:1.0:1775776221.499481:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1775776221.499482:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775776221.499485:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776221.499485:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776221.499492:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1775776221.499493:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776221.499496:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775776221.499497:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1775776221.499498:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776221.499502:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776221.499504:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775776221.499505:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775776221.499511:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00100000:1.0:1775776221.499512:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e063b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710398080:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776221.499513:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776221.499517:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776221.499521:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776221.499524:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776221.499526:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776221.499529:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776221.499530:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776221.499533:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776221.499534:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:0.0:1775776221.499539:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000010:1.0:1775776221.499542:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 00000008:00000001:0.0:1775776221.499543:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776221.499546:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776221.499548:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776221.499554:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061f80. 00000400:00000200:1.0:1775776221.499555:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000040:0.0:1775776221.499559:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000400:00000010:1.0:1775776221.499562:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 02000000:00000001:0.0:1775776221.499563:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776221.499566:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776221.499569:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776221.499571:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1775776221.499572:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 02000000:00000001:0.0:1775776221.499573:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776221.499580:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4800. 00000100:00000200:1.0:1775776221.499580:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710398080, portal 25 00000100:00000001:1.0:1775776221.499585:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776221.499586:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776221.499588:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574568320 : -123823134983296 : ffff8f622e063b80) 00000100:00100000:0.0:1775776221.499593:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061f80 x1862022710398208/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1775776221.499598:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e063b80 x1862022710398080/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776221.499607:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776221.499611:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776221.499618:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776221.499619:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000029d3794] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061f80 x1862022710398208/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776221.499626:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776221.499632:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000001:0.0:1775776221.499633:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776221.499638:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1775776221.499639:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710398080, offset 0 00000100:00080000:0.0:1775776221.499644:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776221.499646:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776221.499647:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776221.499653:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775776221.499653:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00080000:0.0:1775776221.499658:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13104 00000400:00000200:0.0:1775776221.499661:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776221.499664:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776221.499665:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776221.499667:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776221.499668:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776221.499670:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776221.499673:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776221.499674:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000001:0.0:1775776221.499677:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776221.499679:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776221.499681:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776221.499683:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000400:00000200:1.0:1775776221.499684:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000020:00000001:0.0:1775776221.499685:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775776221.499690:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775776221.499694:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776221.499694:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1775776221.499696:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776221.499699:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776221.499700:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776221.499703:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000020:00000040:0.0:1775776221.499704:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775776221.499709:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776221.499713:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776221.499713:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 02000000:00000001:0.0:1775776221.499718:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776221.499720:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 02000000:00000001:0.0:1775776221.499722:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776221.499726:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776221.499728:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453662:453662:256:4294967295] 192.168.201.115@tcp LPNI seq info [453587:453587:8:4294967295] 00000100:00000001:0.0:1775776221.499729:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776221.499731:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776221.499736:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776221.499738:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776221.499742:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061180. 00000400:00000200:1.0:1775776221.499742:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000040:0.0:1775776221.499746:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776221.499748:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776221.499750:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776221.499753:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776221.499755:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776221.499758:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776221.499758:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 02000000:00000010:0.0:1775776221.499764:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7c00. 00000100:00000001:0.0:1775776221.499767:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776221.499768:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00100000:0.0:1775776221.499773:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061180 x1862022710398336/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:1.0:1775776221.499775:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000100:00000001:0.0:1775776221.499781:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776221.499783:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776221.499787:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:0.0:1775776221.499791:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e23859] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061180 x1862022710398336/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1775776221.499792:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775776221.499802:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775776221.499804:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776221.499807:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775776221.499807:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000100:00000040:0.0:1775776221.499813:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000100:1.0:1775776221.499814:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000040:0.0:1775776221.499817:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:1.0:1775776221.499818:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000100:00000040:0.0:1775776221.499823:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1775776221.499826:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:0.0:1775776221.499827:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13114) 00000400:00000200:1.0:1775776221.499831:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776221.499835:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776221.499837:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776221.499843:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776221.499846:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e063b80 x1862022710398080/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.499854:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776221.499856:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e063b80 x1862022710398080/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776246 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.499861:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776221.499862:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776221.499867:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776221.499869:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776221.499871:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307780358 00000100:00000010:1.0:1775776221.499875:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2600. 00000100:00000001:1.0:1775776221.499880:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776221.499885:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307780358 00000100:00000010:1.0:1775776221.499887:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2420. 00000100:00000001:1.0:1775776221.499890:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776221.499892:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776221.499894:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776221.499896:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776221.499899:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776221/real 1775776221] req@ffff8f622e063b80 x1862022710398080/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776221.499905:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776221.499906:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776221.499908:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776221.499910:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776221.499911:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776221.499912:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776221.499914:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e063b80 x1862022710398080/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776221.499920:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e063b80 x1862022710398080/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.499925:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776221.499926:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776221.499927:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776221.499929:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776221.499931:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e063b80 x1862022710398080/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.499935:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776221.499938:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e063b80 x1862022710398080/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.499943:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776221.499944:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776221.499945:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776221.499946:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776221.499948:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776221.499950:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776221.499952:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776221.499954:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776221.499956:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776221.499959:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e063b80 x1862022710398080/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776221.499964:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e063b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710398080:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776221.499967:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776221.499968:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776221.499969:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e063b80 x1862022710398080/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.499974:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776221.499976:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776221.499978:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776221.499980:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776221.499981:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776221.499982:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776221.499984:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776221.499985:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e5c00. 02000000:00000001:1.0:1775776221.499988:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776221.499989:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776221.499990:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e063b80. 00000100:00000001:1.0:1775776221.499993:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776221.499993:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776221.499995:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776221.500000:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776221.500001:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776221.500003:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776221.500004:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776221.500005:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776221.500007:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061f80 x1862022710398208/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.500016:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776221.500016:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776221.500018:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776221.500019:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776221.500021:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776221.500022:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710398208:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776221.500024:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776221.500025:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776221.500026:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776221.500027:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776221.500029:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776221.500031:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776221.500035:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776221.500038:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776221.500042:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776221.500044:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710398208, portal 10 00000100:00000001:1.0:1775776221.500047:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776221.500048:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574561152 : -123823134990464 : ffff8f622e061f80) 00000100:00000040:1.0:1775776221.500055:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061f80 x1862022710398208/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.500062:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776221.500065:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776221.500067:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776221.500069:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710398208, offset 0 00000400:00000200:1.0:1775776221.500073:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776221.500078:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776221.500081:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776221.500084:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776221.500087:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776221.500089:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776221.500092:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776221.500096:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453663:453663:256:4294967295] 192.168.201.115@tcp LPNI seq info [453588:453588:8:4294967295] 00000400:00000200:1.0:1775776221.500100:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776221.500107:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776221.500113:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776221.500117:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776221.500120:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776221.500122:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775776221.500126:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776221.500127:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775776221.500131:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776221.500132:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776221.500150:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776221.500153:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776221.500155:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776221.500156:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776221.500158:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776221.500161:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061f80 x1862022710398208/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.500167:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776221.500169:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061f80 x1862022710398208/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776246 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.500173:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776221.500174:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776221.500176:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776221.500179:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776221.500182:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776221.500183:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307776524 00000100:00000001:1.0:1775776221.500185:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307780358 00000100:00000001:1.0:1775776221.500187:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307776524 00000100:00000001:1.0:1775776221.500188:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307780358 00000100:00000001:1.0:1775776221.500190:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776221.500191:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776221.500193:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776221.500195:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776221/real 1775776221] req@ffff8f622e061f80 x1862022710398208/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776221.500201:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776221.500201:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776221.500202:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776221.500204:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776221.500205:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776221.500206:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776221.500208:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061f80 x1862022710398208/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776221.500213:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061f80 x1862022710398208/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.500217:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776221.500218:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776221.500219:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776221.500220:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776221.500222:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061f80 x1862022710398208/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.500226:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776221.500228:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061f80 x1862022710398208/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.500232:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776221.500234:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776221.500235:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776221.500236:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776221.500238:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776221.500241:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776221.500243:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776221.500246:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776221.500247:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776221.500249:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061f80 x1862022710398208/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776221.500254:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710398208:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776221.500256:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776221.500258:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061180 x1862022710398336/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.500262:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776221.500263:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776221.500264:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776221.500266:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776221.500267:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776221.500269:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710398336:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776221.500271:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776221.500272:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776221.500273:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776221.500274:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776221.500277:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776221.500278:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776221.500280:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776221.500282:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776221.500284:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776221.500286:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710398336, portal 10 00000100:00000001:1.0:1775776221.500300:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776221.500302:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574557568 : -123823134994048 : ffff8f622e061180) 00000100:00000040:1.0:1775776221.500307:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061180 x1862022710398336/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.500311:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776221.500313:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776221.500315:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776221.500317:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710398336, offset 0 00000400:00000200:1.0:1775776221.500319:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776221.500322:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776221.500324:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776221.500326:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776221.500328:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776221.500330:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776221.500331:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776221.500333:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453664:453664:256:4294967295] 192.168.201.115@tcp LPNI seq info [453589:453589:8:4294967295] 00000400:00000200:1.0:1775776221.500336:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776221.500340:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776221.500343:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776221.500345:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776221.500348:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776221.500350:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775776221.500352:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776221.500354:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775776221.500356:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776221.500358:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776221.500360:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776221.500362:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776221.500364:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776221.500365:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776221.500367:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776221.500369:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061180 x1862022710398336/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.500374:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776221.500376:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061180 x1862022710398336/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776246 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.500380:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776221.500382:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776221.500384:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776221.500386:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776221.500388:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776221.500389:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307776524 00000100:00000001:1.0:1775776221.500391:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307780358 00000100:00000001:1.0:1775776221.500393:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307776524 00000100:00000001:1.0:1775776221.500394:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307780358 00000100:00000001:1.0:1775776221.500395:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776221.500396:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776221.500397:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776221.500400:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776221/real 1775776221] req@ffff8f622e061180 x1862022710398336/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776221.500404:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776221.500405:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776221.500406:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776221.500408:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776221.500409:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776221.500410:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776221.500412:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061180 x1862022710398336/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776221.500420:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061180 x1862022710398336/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.500427:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776221.500428:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776221.500430:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776221.500431:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776221.500435:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061180 x1862022710398336/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.500442:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776221.500447:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061180 x1862022710398336/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.500454:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776221.500457:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776221.500458:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776221.500460:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776221.500461:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776221.500462:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776221.500464:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776221.500466:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776221.500468:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776221.500470:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061180 x1862022710398336/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776221.500476:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710398336:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776221.500478:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776221.500479:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776221.500481:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061f80 x1862022710398208/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.500485:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776221.500486:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776221.500487:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776221.500489:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776221.500490:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776221.500491:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776221.500492:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776221.500493:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:1.0:1775776221.500494:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776221.500495:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776221.500496:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061f80. 00000100:00000001:1.0:1775776221.500499:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776221.500500:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776221.500501:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776221.500503:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061180 x1862022710398336/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776246 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776221.500508:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776221.500508:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776221.500509:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776221.500510:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776221.500511:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776221.500512:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776221.500513:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776221.500514:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e7c00. 02000000:00000001:1.0:1775776221.500515:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776221.500515:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776221.500516:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061180. 00000100:00000001:1.0:1775776221.500517:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776221.500518:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776221.500519:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776221.500522:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776221.500523:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776221.500524:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776221.500525:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776221.627261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776221.627267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776221.627273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776221.627274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776221.627278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776221.627280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776221.627379:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776221.627383:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776221.627387:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776222.523254:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776222.523262:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776222.523264:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776222.523295:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776222.523300:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776222.523304:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776222.523307:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776222.523309:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776222.523312:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776222.651264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776222.651297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776222.651305:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776222.651306:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776222.651311:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776222.651312:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776222.651385:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776222.651389:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776222.651392:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776223.547236:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776223.547242:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776223.547244:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776223.547248:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776223.547251:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776223.547270:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776223.547274:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776223.547276:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776223.547278:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776223.675296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776223.675303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776223.675309:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776223.675310:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776223.675314:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776223.675315:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776223.675464:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776223.675466:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776223.675469:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776224.571262:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776224.571270:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776224.571271:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776224.571277:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776224.571281:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776224.571284:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776224.571286:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776224.571288:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776224.571291:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776224.699271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776224.699275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776224.699279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776224.699280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776224.699282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776224.699283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776224.699353:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776224.699357:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776224.699361:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776225.403258:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307780358, at 4307780416 00000100:00000010:1.0:1775776225.403271:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2420. 00000100:00000040:0.0:1775776225.403281:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307780358, at 4307780416 00000100:00000010:0.0:1775776225.403290:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2600. 00000100:00000001:1.0:1775776225.595267:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776225.595273:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776225.595274:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776225.595278:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776225.595281:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776225.595284:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776225.595286:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776225.595288:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776225.595290:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776225.723257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776225.723263:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776225.723265:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776225.723267:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776225.723271:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776225.723272:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776225.723274:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776225.723275:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776225.723278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776225.723279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776225.723282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776225.723282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776225.723345:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776225.723348:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776225.723350:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776226.619242:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776226.619252:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1775776226.619260:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776226.619261:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776226.619268:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1775776226.619272:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776226.619272:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776226.619277:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776226.619278:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776226.619283:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776226.619284:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776226.619286:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776226.619289:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776226.619292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776226.619294:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13109 00000400:00000200:2.0:1775776226.619304:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776226.619311:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776226.619314:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776226.619318:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776226.619321:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776226.619323:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776226.619326:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776226.619330:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776226.619331:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776226.619334:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776226.619338:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776226.619340:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776226.619342:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776226.619345:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776226.619347:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776226.619349:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776226.619350:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776226.619353:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776226.619356:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776226.619359:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776226.619362:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776226.619365:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776226.619369:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776226.619371:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776226.619373:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776226.619383:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775776226.619386:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776226.619390:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776226.619392:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776226.619394:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776226.619397:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776226.619402:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776226.619410:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775776226.619417:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776226.619427:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710398464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776226.619437:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776226.619442:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776226.619446:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710398464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776226.619467:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776226.619470:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776226.619474:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776226.619475:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776226.619477:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776226.619480:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776226.619483:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13109 00000400:00000200:2.0:1775776226.619488:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776226.619493:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776226.619495:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776226.619496:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1775776226.619500:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776226.619501:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776226.619505:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1775776226.619505:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776226.619511:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776226.619515:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:1.0:1775776226.619516:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344aa00 x1862022710398464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775776226.619520:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776226.619523:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776226.619527:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776226.619531:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776226.619534:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776226.619537:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776226.619540:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:2.0:1775776226.619541:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776226.619544:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776226.619546:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775776226.619549:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:2.0:1775776226.619550:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776226.619553:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776226.619554:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776226.619559:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775776226.619560:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00100000:1.0:1775776226.619569:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710398464:192.168.201.115@tcp:250:kworker.0 00000100:00080000:2.0:1775776226.619569:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776226.619574:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776226.619576:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776226.619577:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776226.619580:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776226.619582:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776226.619584:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1775776226.619584:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776226.619590:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776226.619591:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 00000100:00000001:2.0:1775776226.619592:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776226.619594:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1775776226.619598:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1775776226.619603:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000008:00000001:2.0:1775776226.619606:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776226.619608:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000100:00000001:2.0:1775776226.619609:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776226.619615:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000400:00000010:1.0:1775776226.619616:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000020:00000040:2.0:1775776226.619619:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000200:1.0:1775776226.619623:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710398464, portal 25 02000000:00000001:2.0:1775776226.619623:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776226.619625:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776226.619627:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776226.619629:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662545920 : -123823047005696 : ffff8f623344aa00) 00000100:00000001:2.0:1775776226.619629:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776226.619632:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776226.619635:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776226.619637:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344aa00 x1862022710398464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1775776226.619643:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:1.0:1775776226.619644:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776226.619648:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:2.0:1775776226.619650:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776226.619651:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776226.619655:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710398464, offset 0 00000100:00100000:2.0:1775776226.619660:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710398592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776226.619663:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:2.0:1775776226.619672:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775776226.619673:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:2.0:1775776226.619677:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776226.619680:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000040:2.0:1775776226.619686:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710398592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776226.619687:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776226.619691:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776226.619695:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776226.619699:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776226.619706:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453665:453665:256:4294967295] 192.168.201.115@tcp LPNI seq info [453590:453590:8:4294967295] 00000400:00000200:1.0:1775776226.619716:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:2.0:1775776226.619717:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776226.619722:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776226.619728:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00080000:2.0:1775776226.619730:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000200:1.0:1775776226.619734:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:2.0:1775776226.619736:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776226.619739:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000010:1.0:1775776226.619740:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000100:00000001:2.0:1775776226.619742:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776226.619746:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13109 00000800:00000200:1.0:1775776226.619749:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776226.619754:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000400:00000200:2.0:1775776226.619757:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000200:1.1:1775776226.619762:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000400:00000200:2.0:1775776226.619766:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000800:00000200:1.1:1775776226.619767:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00080000:2.0:1775776226.619771:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775776226.619774:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776226.619777:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000100:00000040:2.0:1775776226.619779:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776226.619782:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1775776226.619783:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776226.619788:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776226.619791:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1775776226.619793:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776226.619796:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776226.619798:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000020:00000001:2.0:1775776226.619798:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776226.619802:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000100:00000001:1.0:1775776226.619805:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:2.0:1775776226.619806:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776226.619809:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000100:00000200:1.0:1775776226.619810:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344aa00 x1862022710398464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775776226.619813:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776226.619820:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000001:1.0:1775776226.619826:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776226.619828:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000040:1.0:1775776226.619832:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344aa00 x1862022710398464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776251 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776226.619833:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776226.619838:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776226.619843:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776226.619848:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00000001:1.0:1775776226.619849:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776226.619854:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:2.0:1775776226.619854:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776226.619859:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776226.619861:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776226.619862:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776226.619865:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775776226.619869:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:2.0:1775776226.619871:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776226.619874:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307785478 00000100:00000001:2.0:1775776226.619874:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776226.619878:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000010:1.0:1775776226.619882:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2420. 00000008:00000001:2.0:1775776226.619882:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776226.619886:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776226.619891:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000010:2.0:1775776226.619891:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775776226.619897:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000100:00000001:1.0:1775776226.619900:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307785478 02000000:00000001:2.0:1775776226.619900:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776226.619903:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776226.619906:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2360. 00000100:00000001:2.0:1775776226.619906:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776226.619909:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775776226.619911:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1775776226.619912:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776226.619914:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776226.619918:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776226.619919:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:1.0:1775776226.619920:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1775776226.619925:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776226.619926:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776226/real 1775776226] req@ffff8f623344aa00 x1862022710398464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776226.619935:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710398720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1775776226.619937:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776226.619940:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776226.619943:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776226.619947:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776226.619950:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:2.0:1775776226.619953:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776226.619954:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1775776226.619958:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776226.619959:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344aa00 x1862022710398464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775776226.619968:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710398720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776226.619972:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344aa00 x1862022710398464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776226.619998:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776226.620001:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776226.620001:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776226.620005:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776226.620008:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776226.620008:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775776226.620013:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344aa00 x1862022710398464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775776226.620016:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776226.620023:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776226.620024:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776226.620030:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344aa00 x1862022710398464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775776226.620030:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776226.620039:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:2.0:1775776226.620039:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13119) 00000100:00000001:1.0:1775776226.620042:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776226.620044:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776226.620046:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776226.620048:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776226.620050:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776226.620052:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776226.620055:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776226.620057:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776226.620061:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344aa00 x1862022710398464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776226.620067:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710398464:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776226.620070:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776226.620071:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776226.620073:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344aa00 x1862022710398464/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776226.620077:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776226.620079:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776226.620081:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776226.620083:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776226.620084:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776226.620085:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776226.620087:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776226.620088:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4f000. 02000000:00000001:1.0:1775776226.620091:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776226.620092:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776226.620094:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344aa00. 00000100:00000001:1.0:1775776226.620096:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776226.620097:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776226.620098:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776226.620104:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776226.620105:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776226.620106:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776226.620107:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776226.620108:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776226.620110:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233448380 x1862022710398592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776226.620115:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776226.620116:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776226.620118:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776226.620119:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776226.620120:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776226.620122:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233448380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710398592:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776226.620124:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776226.620125:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776226.620126:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776226.620127:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776226.620129:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776226.620130:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776226.620151:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776226.620153:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776226.620156:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776226.620158:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710398592, portal 10 00000100:00000001:1.0:1775776226.620159:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776226.620160:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662536064 : -123823047015552 : ffff8f6233448380) 00000100:00000040:1.0:1775776226.620165:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233448380 x1862022710398592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776226.620169:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776226.620171:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776226.620172:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776226.620174:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710398592, offset 0 00000400:00000200:1.0:1775776226.620177:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776226.620184:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776226.620187:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776226.620189:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776226.620191:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776226.620192:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776226.620194:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776226.620197:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453666:453666:256:4294967295] 192.168.201.115@tcp LPNI seq info [453591:453591:8:4294967295] 00000400:00000200:1.0:1775776226.620215:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776226.620222:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776226.620227:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776226.620231:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776226.620235:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776226.620238:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775776226.620243:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776226.620245:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775776226.620249:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776226.620251:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776226.620255:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776226.620259:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776226.620262:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776226.620263:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776226.620267:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776226.620269:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233448380 x1862022710398592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776226.620275:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776226.620277:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448380 x1862022710398592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776251 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776226.620282:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776226.620283:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776226.620285:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776226.620288:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776226.620291:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776226.620293:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307781644 00000100:00000001:1.0:1775776226.620295:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307785478 00000100:00000001:1.0:1775776226.620296:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307781644 00000100:00000001:1.0:1775776226.620298:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307785478 00000100:00000001:1.0:1775776226.620299:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776226.620301:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776226.620303:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776226.620305:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776226/real 1775776226] req@ffff8f6233448380 x1862022710398592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776226.620311:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776226.620312:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776226.620313:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776226.620314:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776226.620315:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776226.620316:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776226.620318:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233448380 x1862022710398592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776226.620324:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233448380 x1862022710398592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776226.620328:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776226.620329:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776226.620330:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776226.620331:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776226.620333:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233448380 x1862022710398592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776226.620340:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776226.620344:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233448380 x1862022710398592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776226.620351:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776226.620354:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776226.620355:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776226.620357:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776226.620359:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776226.620364:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776226.620367:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776226.620371:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776226.620373:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776226.620378:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233448380 x1862022710398592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776226.620386:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233448380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710398592:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776226.620389:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776226.620391:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449f80 x1862022710398720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776226.620395:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776226.620396:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776226.620398:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776226.620399:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776226.620401:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776226.620402:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710398720:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776226.620404:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776226.620406:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776226.620407:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776226.620408:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776226.620410:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776226.620412:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776226.620414:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776226.620416:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776226.620418:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776226.620420:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710398720, portal 10 00000100:00000001:1.0:1775776226.620421:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776226.620422:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662543232 : -123823047008384 : ffff8f6233449f80) 00000100:00000040:1.0:1775776226.620426:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449f80 x1862022710398720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776226.620431:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776226.620433:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776226.620434:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776226.620436:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710398720, offset 0 00000400:00000200:1.0:1775776226.620438:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776226.620441:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776226.620444:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776226.620446:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776226.620447:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776226.620449:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776226.620451:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776226.620453:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453667:453667:256:4294967295] 192.168.201.115@tcp LPNI seq info [453592:453592:8:4294967295] 00000400:00000200:1.0:1775776226.620455:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776226.620460:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776226.620462:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776226.620464:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776226.620467:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776226.620469:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775776226.620472:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776226.620473:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775776226.620475:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776226.620477:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776226.620479:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776226.620481:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776226.620483:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776226.620484:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776226.620486:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776226.620488:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449f80 x1862022710398720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776226.620494:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776226.620495:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449f80 x1862022710398720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776251 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776226.620500:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776226.620502:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776226.620503:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776226.620505:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776226.620507:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776226.620509:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307781644 00000100:00000001:1.0:1775776226.620510:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307785478 00000100:00000001:1.0:1775776226.620512:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307781644 00000100:00000001:1.0:1775776226.620513:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307785478 00000100:00000001:1.0:1775776226.620514:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776226.620516:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776226.620517:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776226.620519:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776226/real 1775776226] req@ffff8f6233449f80 x1862022710398720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776226.620523:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776226.620524:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776226.620525:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776226.620527:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776226.620528:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776226.620529:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776226.620531:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449f80 x1862022710398720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776226.620536:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449f80 x1862022710398720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776226.620539:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776226.620540:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776226.620541:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776226.620542:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776226.620544:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449f80 x1862022710398720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776226.620548:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776226.620551:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449f80 x1862022710398720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776226.620556:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776226.620557:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776226.620558:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776226.620559:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776226.620560:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776226.620561:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776226.620563:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776226.620565:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776226.620566:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776226.620568:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449f80 x1862022710398720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776226.620573:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710398720:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776226.620575:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776226.620576:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776226.620577:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233448380 x1862022710398592/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776226.620581:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776226.620582:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776226.620584:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776226.620585:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776226.620586:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776226.620587:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776226.620589:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776226.620590:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4ec00. 02000000:00000001:1.0:1775776226.620592:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776226.620593:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776226.620594:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233448380. 00000100:00000001:1.0:1775776226.620595:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776226.620596:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776226.620598:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776226.620600:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449f80 x1862022710398720/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776251 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776226.620604:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776226.620605:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776226.620605:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776226.620607:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776226.620607:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776226.620608:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776226.620609:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776226.620610:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4e000. 02000000:00000001:1.0:1775776226.620611:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776226.620612:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776226.620613:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449f80. 00000100:00000001:1.0:1775776226.620614:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776226.620615:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776226.620616:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776226.620618:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776226.620619:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776226.620620:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776226.620621:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776226.747302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776226.747309:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776226.747316:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776226.747317:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776226.747322:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776226.747323:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776226.747391:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776226.747395:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776226.747398:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776227.643294:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776227.643301:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776227.643303:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776227.643310:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776227.643314:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776227.643318:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776227.643321:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776227.643325:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776227.643327:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776227.771262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776227.771267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776227.771272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776227.771273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776227.771276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776227.771277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776227.771346:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776227.771350:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776227.771354:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776228.667221:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776228.667227:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776228.667228:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776228.667232:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776228.667235:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776228.667238:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776228.667241:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776228.667243:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776228.667245:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776228.795278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776228.795292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776228.795298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776228.795300:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776228.795304:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776228.795305:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776228.795362:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776228.795366:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776228.795368:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776229.691279:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776229.691286:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776229.691288:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776229.691294:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776229.691298:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776229.691302:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776229.691304:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776229.691306:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776229.691310:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776229.819252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776229.819257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776229.819266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776229.819269:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776229.819271:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776229.819275:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776229.819278:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776229.819280:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776229.819282:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776229.819284:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776229.819288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776229.819290:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776229.819291:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776229.819293:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776229.819294:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776229.819295:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776229.819297:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776229.819298:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776229.819372:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776229.819376:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776229.819379:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776230.523250:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307785478, at 4307785536 00000100:00000010:1.0:1775776230.523261:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2360. 00000100:00000040:0.0:1775776230.523266:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307785478, at 4307785536 00000100:00000010:0.0:1775776230.523276:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2420. 00000100:00000001:1.0:1775776230.715281:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776230.715289:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776230.715291:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776230.715297:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776230.715301:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776230.715305:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776230.715309:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776230.715311:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776230.715314:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776230.843263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776230.843269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776230.843275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776230.843276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776230.843281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776230.843282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776230.843344:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776230.843348:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776230.843350:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776231.739244:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776231.739251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776231.739254:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776231.739259:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776231.739264:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776231.739268:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776231.739271:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776231.739274:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775776231.739275:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776231.739278:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776231.739285:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776231.739288:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776231.739292:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776231.739296:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776231.739305:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13114 00000400:00000200:3.0:1775776231.739315:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776231.739323:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776231.739326:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776231.739330:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776231.739333:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776231.739336:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776231.739338:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776231.739342:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776231.739344:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776231.739347:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776231.739349:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776231.739352:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776231.739355:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776231.739358:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776231.739360:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776231.739362:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776231.739364:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776231.739367:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776231.739371:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776231.739382:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776231.739386:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776231.739389:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776231.739396:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776231.739399:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776231.739401:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776231.739410:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73100. 00000020:00000040:3.0:1775776231.739414:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776231.739419:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776231.739421:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776231.739424:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776231.739427:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776231.739432:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776231.739442:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000100:00000001:3.0:1775776231.739449:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776231.739458:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73100 x1862022710398848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776231.739468:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776231.739471:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776231.739475:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f67e6552] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73100 x1862022710398848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776231.739490:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776231.739493:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776231.739495:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776231.739497:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776231.739498:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776231.739500:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776231.739502:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13114 00000400:00000200:3.0:1775776231.739508:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776231.739519:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1775776231.739519:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:1.0:1775776231.739523:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776231.739524:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776231.739527:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776231.739531:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776231.739535:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776231.739538:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73100 x1862022710398848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776231.739540:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776231.739546:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776231.739552:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776231.739555:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:1.0:1775776231.739556:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776231.739559:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775776231.739559:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776231.739563:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 02000000:00000001:1.0:1775776231.739564:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1775776231.739567:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776231.739569:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775776231.739571:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776231.739575:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776231.739578:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00100000:1.0:1775776231.739583:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710398848:192.168.201.115@tcp:250:kworker.0 00000100:00000001:3.0:1775776231.739584:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776231.739590:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776231.739595:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1775776231.739596:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776231.739598:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776231.739600:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776231.739603:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:3.0:1775776231.739605:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000010:1.0:1775776231.739610:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 00000100:00080000:3.0:1775776231.739610:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000001:1.0:1775776231.739615:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776231.739616:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776231.739620:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776231.739624:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776231.739626:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1775776231.739629:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 02000000:00000001:3.0:1775776231.739630:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776231.739635:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000010:1.0:1775776231.739638:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000008:00100000:3.0:1775776231.739639:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776231.739645:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776231.739647:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710398848, portal 25 00000100:00000001:3.0:1775776231.739647:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776231.739653:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776231.739655:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849016576 : -123823860535040 : ffff8f6202c73100) 00000100:00000010:3.0:1775776231.739657:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775776231.739661:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000040:1.0:1775776231.739663:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73100 x1862022710398848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776256 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775776231.739665:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776231.739668:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776231.739672:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776231.739676:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776231.739679:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776231.739682:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000010:3.0:1775776231.739685:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000001:3.0:1775776231.739690:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776231.739695:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776231.739699:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00100000:3.0:1775776231.739701:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710398976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1775776231.739704:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710398848, offset 0 00000400:00000200:1.0:1775776231.739716:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776231.739716:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776231.739721:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776231.739727:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775776231.739729:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710398976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776231.739736:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:3.0:1775776231.739746:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776231.739747:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:3.0:1775776231.739751:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776231.739755:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00080000:3.0:1775776231.739758:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:1.0:1775776231.739763:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1775776231.739764:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776231.739768:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775776231.739771:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:3.0:1775776231.739774:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775776231.739778:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453668:453668:256:4294967295] 192.168.201.115@tcp LPNI seq info [453593:453593:8:4294967295] 00000100:00080000:3.0:1775776231.739781:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13114 00000400:00000200:1.0:1775776231.739790:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776231.739791:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776231.739796:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776231.739799:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776231.739804:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776231.739808:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776231.739811:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000040:3.0:1775776231.739813:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776231.739818:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776231.739822:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000800:00000200:1.0:1775776231.739825:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000020:00000040:3.0:1775776231.739825:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776231.739830:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776231.739833:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000800:00000010:1.0:1775776231.739834:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000020:00000001:3.0:1775776231.739836:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776231.739839:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:1.0:1775776231.739844:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776231.739845:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000800:00000200:1.0:1775776231.739850:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000100:00000001:3.0:1775776231.739850:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776231.739853:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776231.739858:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776231.739863:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000800:00000200:1.1:1775776231.739863:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776231.739870:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000100:00080000:3.0:1775776231.739871:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776231.739877:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1775776231.739880:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776231.739883:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776231.739889:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:3.0:1775776231.739889:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776231.739894:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775776231.739897:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 02000000:00000001:3.0:1775776231.739899:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776231.739903:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775776231.739904:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000008:00100000:3.0:1775776231.739905:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000010:1.0:1775776231.739908:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000008:00000001:3.0:1775776231.739910:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776231.739913:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776231.739916:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:3.0:1775776231.739918:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000100:00000200:1.0:1775776231.739922:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73100 x1862022710398848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776256 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775776231.739923:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776231.739929:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776231.739931:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776231.739936:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776231.739939:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776231.739942:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776231.739943:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776231.739948:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73100 x1862022710398848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776256 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1775776231.739948:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000001:3.0:1775776231.739955:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776231.739962:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776231.739964:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710399104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776231.739966:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776231.739972:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776231.739973:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776231.739974:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776231.739980:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710399104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776231.739983:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776231.739989:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307790598 00000100:00000001:3.0:1775776231.739990:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776231.739994:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776231.739998:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000010:1.0:1775776231.739999:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2360. 00000100:00000040:3.0:1775776231.740000:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776231.740004:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776231.740008:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000040:3.0:1775776231.740008:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13124) 00000100:00000001:1.0:1775776231.740018:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307790598 00000100:00000010:1.0:1775776231.740022:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e22a0. 00000100:00000001:1.0:1775776231.740026:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776231.740030:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776231.740033:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776231.740036:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776231.740041:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776231/real 1775776231] req@ffff8f6202c73100 x1862022710398848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776231.740051:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776231.740053:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776231.740056:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776231.740059:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776231.740061:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776231.740063:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776231.740068:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73100 x1862022710398848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776231.740077:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73100 x1862022710398848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776231.740084:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776231.740086:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776231.740089:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776231.740091:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776231.740095:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73100 x1862022710398848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776231.740102:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776231.740107:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73100 x1862022710398848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776231.740160:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776231.740163:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776231.740164:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776231.740166:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776231.740168:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776231.740171:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776231.740173:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776231.740177:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776231.740180:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776231.740184:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73100 x1862022710398848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776231.740192:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710398848:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776231.740197:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776231.740199:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776231.740201:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73100 x1862022710398848/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776231.740208:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776231.740210:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776231.740213:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776231.740216:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776231.740218:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776231.740220:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776231.740222:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776231.740224:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34400. 02000000:00000001:1.0:1775776231.740229:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776231.740230:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776231.740232:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73100. 00000100:00000001:1.0:1775776231.740235:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776231.740236:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776231.740239:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776231.740245:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776231.740247:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776231.740249:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776231.740250:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776231.740252:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776231.740255:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70000 x1862022710398976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776231.740262:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776231.740264:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776231.740266:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776231.740267:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776231.740270:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776231.740273:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710398976:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776231.740276:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776231.740278:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776231.740280:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776231.740282:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776231.740285:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776231.740287:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776231.740291:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776231.740294:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776231.740298:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776231.740301:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710398976, portal 10 00000100:00000001:1.0:1775776231.740303:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776231.740305:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004032 : -123823860547584 : ffff8f6202c70000) 00000100:00000040:1.0:1775776231.740312:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70000 x1862022710398976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776256 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776231.740318:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776231.740321:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776231.740323:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776231.740326:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710398976, offset 0 00000400:00000200:1.0:1775776231.740330:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776231.740335:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776231.740339:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776231.740342:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776231.740344:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776231.740347:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776231.740349:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776231.740353:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453669:453669:256:4294967295] 192.168.201.115@tcp LPNI seq info [453594:453594:8:4294967295] 00000400:00000200:1.0:1775776231.740358:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776231.740364:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776231.740369:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776231.740382:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776231.740387:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776231.740393:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775776231.740398:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776231.740401:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1775776231.740405:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776231.740407:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776231.740410:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776231.740414:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776231.740418:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776231.740419:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776231.740423:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776231.740426:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70000 x1862022710398976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776256 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776231.740434:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776231.740437:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70000 x1862022710398976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776256 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776231.740444:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776231.740445:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776231.740449:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776231.740452:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776231.740456:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776231.740459:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307786764 00000100:00000001:1.0:1775776231.740462:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307790598 00000100:00000001:1.0:1775776231.740464:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307786764 00000100:00000001:1.0:1775776231.740466:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307790598 00000100:00000001:1.0:1775776231.740469:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776231.740472:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776231.740474:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776231.740478:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776231/real 1775776231] req@ffff8f6202c70000 x1862022710398976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776231.740486:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776231.740487:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776231.740489:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776231.740491:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776231.740492:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776231.740495:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776231.740498:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70000 x1862022710398976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776231.740506:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70000 x1862022710398976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776231.740512:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776231.740514:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776231.740515:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776231.740517:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776231.740521:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70000 x1862022710398976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776231.740527:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776231.740531:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70000 x1862022710398976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776231.740538:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776231.740540:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776231.740541:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776231.740543:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776231.740545:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776231.740550:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776231.740553:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776231.740557:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776231.740560:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776231.740564:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70000 x1862022710398976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776231.740571:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710398976:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776231.740575:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776231.740579:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70e00 x1862022710399104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776231.740584:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776231.740586:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776231.740588:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776231.740590:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776231.740592:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776231.740595:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710399104:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776231.740598:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776231.740600:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776231.740602:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776231.740604:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776231.740607:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776231.740609:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776231.740612:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776231.740615:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776231.740619:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776231.740622:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710399104, portal 10 00000100:00000001:1.0:1775776231.740624:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776231.740625:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849007616 : -123823860544000 : ffff8f6202c70e00) 00000100:00000040:1.0:1775776231.740631:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70e00 x1862022710399104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776256 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776231.740637:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776231.740640:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776231.740642:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776231.740645:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710399104, offset 0 00000400:00000200:1.0:1775776231.740649:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776231.740654:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776231.740658:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776231.740661:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776231.740664:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776231.740666:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776231.740669:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776231.740672:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453670:453670:256:4294967295] 192.168.201.115@tcp LPNI seq info [453595:453595:8:4294967295] 00000400:00000200:1.0:1775776231.740677:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776231.740683:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776231.740688:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776231.740691:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776231.740696:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776231.740699:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775776231.740703:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776231.740705:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1775776231.740710:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776231.740712:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776231.740715:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776231.740718:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776231.740721:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776231.740723:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776231.740726:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776231.740729:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70e00 x1862022710399104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776256 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776231.740737:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776231.740741:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70e00 x1862022710399104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776256 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776231.740747:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776231.740749:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776231.740752:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776231.740755:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776231.740758:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776231.740760:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307786764 00000100:00000001:1.0:1775776231.740763:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307790598 00000100:00000001:1.0:1775776231.740765:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307786764 00000100:00000001:1.0:1775776231.740767:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307790598 00000100:00000001:1.0:1775776231.740769:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776231.740772:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776231.740774:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776231.740778:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776231/real 1775776231] req@ffff8f6202c70e00 x1862022710399104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776231.740785:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776231.740787:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776231.740788:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776231.740791:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776231.740792:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776231.740795:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776231.740798:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70e00 x1862022710399104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776231.740806:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70e00 x1862022710399104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776231.740814:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776231.740815:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776231.740817:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776231.740818:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776231.740822:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70e00 x1862022710399104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776231.740828:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776231.740833:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70e00 x1862022710399104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776231.740840:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776231.740843:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776231.740844:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776231.740846:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776231.740847:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776231.740850:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776231.740852:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776231.740856:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776231.740858:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776231.740862:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70e00 x1862022710399104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776231.740870:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710399104:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776231.740874:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776231.740876:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776231.740879:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70000 x1862022710398976/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776231.740885:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776231.740887:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776231.740888:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776231.740891:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776231.740893:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776231.740894:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776231.740897:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776231.740898:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37000. 02000000:00000001:1.0:1775776231.740901:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776231.740903:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776231.740904:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70000. 00000100:00000001:1.0:1775776231.740907:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776231.740909:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776231.740911:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776231.740914:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70e00 x1862022710399104/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776256 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776231.740921:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776231.740922:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776231.740924:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776231.740926:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776231.740927:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776231.740928:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776231.740930:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776231.740931:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34800. 02000000:00000001:1.0:1775776231.740934:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776231.740935:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776231.740936:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70e00. 00000100:00000001:1.0:1775776231.740938:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776231.740940:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776231.740942:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776231.740945:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776231.740947:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776231.740949:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776231.740950:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776231.867221:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776231.867226:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776231.867231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776231.867232:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776231.867235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776231.867236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776231.867306:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776231.867310:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776231.867313:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776232.763253:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776232.763258:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776232.763260:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776232.763264:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776232.763267:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776232.763270:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776232.763272:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776232.763274:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776232.763276:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776232.891251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776232.891258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776232.891265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776232.891267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776232.891272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776232.891273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776232.891330:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776232.891333:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776232.891336:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776233.787205:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776233.787213:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776233.787215:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776233.787220:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776233.787224:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776233.787228:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776233.787233:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776233.787236:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776233.787239:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776233.915270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776233.915276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776233.915282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776233.915284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776233.915288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776233.915289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776233.915361:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776233.915365:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776233.915369:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776234.811241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776234.811247:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776234.811249:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776234.811254:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776234.811256:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776234.811259:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776234.811261:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776234.811263:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776234.811265:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776234.939226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776234.939232:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776234.939237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776234.939237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776234.939240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776234.939241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776234.939301:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776234.939304:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776234.939307:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776235.643266:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307790598, at 4307790656 00000100:00000040:1.0:1775776235.643273:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307790598, at 4307790656 00000100:00000010:0.0:1775776235.643279:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2360. 00000100:00000010:1.0:1775776235.643282:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e22a0. 00000100:00000001:1.0:1775776235.835251:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776235.835259:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776235.835260:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776235.835267:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776235.835271:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776235.835275:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776235.835278:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776235.835281:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776235.835283:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776235.963247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776235.963255:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776235.963257:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776235.963261:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776235.963266:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776235.963268:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776235.963270:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776235.963272:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776235.963277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776235.963278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776235.963281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776235.963281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776235.963338:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776235.963342:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776235.963345:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776236.859238:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776236.859242:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775776236.859246:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776236.859249:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776236.859250:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776236.859254:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776236.859254:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776236.859261:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00100000:1.0:1775776236.859262:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:3.0:1775776236.859267:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13119 00000100:00000001:1.0:1775776236.859268:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776236.859273:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776236.859278:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:3.0:1775776236.859278:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776236.859283:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1775776236.859286:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:1.0:1775776236.859290:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776236.859290:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776236.859297:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776236.859300:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776236.859302:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776236.859303:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776236.859306:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776236.859307:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776236.859309:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776236.859312:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776236.859313:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776236.859315:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776236.859318:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776236.859320:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776236.859321:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776236.859323:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776236.859325:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776236.859328:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776236.859330:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776236.859332:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776236.859334:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776236.859336:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776236.859337:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776236.859339:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776236.859346:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775776236.859349:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776236.859352:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776236.859353:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776236.859355:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776236.859357:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776236.859360:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776236.859366:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775776236.859371:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776236.859379:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710399232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776236.859387:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776236.859391:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776236.859394:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710399232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776236.859411:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776236.859413:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776236.859415:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776236.859416:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776236.859417:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776236.859419:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776236.859421:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13119 00000400:00000200:3.0:1775776236.859424:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776236.859426:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776236.859428:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776236.859430:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776236.859432:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776236.859434:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776236.859435:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776236.859437:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776236.859438:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776236.859439:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776236.859441:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776236.859443:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776236.859444:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1775776236.859445:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776236.859450:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775776236.859451:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776236.859453:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776236.859455:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776236.859458:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776236.859462:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:1.0:1775776236.859464:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71c00 x1862022710399232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775776236.859468:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776236.859475:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776236.859482:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776236.859488:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776236.859489:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776236.859492:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776236.859494:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776236.859496:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776236.859501:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776236.859504:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776236.859506:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776236.859509:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710399232:192.168.201.115@tcp:250:kworker.0 00000100:00000001:3.0:1775776236.859510:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776236.859516:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775776236.859517:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00000001:3.0:1775776236.859522:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776236.859523:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1775776236.859524:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776236.859527:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776236.859530:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 02000000:00000001:1.0:1775776236.859532:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:3.0:1775776236.859534:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776236.859538:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000010:1.0:1775776236.859540:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:3.0:1775776236.859540:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776236.859545:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776236.859546:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776236.859547:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776236.859552:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776236.859557:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000010:3.0:1775776236.859558:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000400:00000010:1.0:1775776236.859563:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000100:00000001:3.0:1775776236.859563:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776236.859572:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710399360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1775776236.859573:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776236.859584:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710399232, portal 25 00000100:00000001:3.0:1775776236.859585:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776236.859589:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776236.859589:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776236.859593:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849011200 : -123823860540416 : ffff8f6202c71c00) 00000100:00000040:3.0:1775776236.859597:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710399360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776236.859605:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71c00 x1862022710399232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776261 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.859620:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775776236.859623:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776236.859627:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00080000:3.0:1775776236.859629:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:1.0:1775776236.859632:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00080000:3.0:1775776236.859634:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000200:1.0:1775776236.859637:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710399232, offset 0 00000100:00000001:3.0:1775776236.859637:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776236.859640:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776236.859644:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775776236.859649:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00080000:3.0:1775776236.859650:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13119 00000400:00000200:3.0:1775776236.859657:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776236.859660:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776236.859662:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776236.859666:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1775776236.859671:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000040:3.0:1775776236.859672:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776236.859678:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776236.859682:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1775776236.859683:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776236.859687:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776236.859690:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:1.0:1775776236.859691:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000020:00000040:3.0:1775776236.859693:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776236.859696:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:1.0:1775776236.859698:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000020:00000040:3.0:1775776236.859700:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776236.859704:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776236.859706:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000020:00000001:3.0:1775776236.859708:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776236.859715:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776236.859716:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453671:453671:256:4294967295] 192.168.201.115@tcp LPNI seq info [453596:453596:8:4294967295] 00000100:00000001:3.0:1775776236.859719:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776236.859727:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775776236.859736:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776236.859742:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:3.0:1775776236.859746:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776236.859751:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000800:00000200:1.0:1775776236.859755:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00080000:3.0:1775776236.859758:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776236.859762:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775776236.859764:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 02000000:00000001:3.0:1775776236.859764:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776236.859770:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776236.859773:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776236.859777:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775776236.859778:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776236.859782:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000200:1.0:1775776236.859784:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000008:00000001:3.0:1775776236.859786:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776236.859791:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776236.859796:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000010:3.0:1775776236.859797:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000800:00000200:1.1:1775776236.859801:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000020:00000040:3.0:1775776236.859802:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776236.859808:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776236.859811:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1775776236.859812:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776236.859816:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000010:1.0:1775776236.859817:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 02000000:00000001:3.0:1775776236.859818:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1775776236.859825:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:3.0:1775776236.859830:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776236.859831:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 02000000:00000010:3.0:1775776236.859838:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000400:00000200:1.0:1775776236.859840:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000100:00000001:3.0:1775776236.859842:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776236.859844:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00100000:3.0:1775776236.859851:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710399488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776236.859852:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776236.859860:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71c00 x1862022710399232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776261 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776236.859864:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776236.859869:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776236.859879:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710399488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.859881:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776236.859889:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71c00 x1862022710399232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776261 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776236.859897:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776236.859902:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776236.859907:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776236.859908:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776236.859912:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1775776236.859914:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776236.859918:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776236.859920:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776236.859924:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13129) 00000400:00000200:1.0:1775776236.859928:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776236.859933:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307795718 00000100:00000010:1.0:1775776236.859938:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e22a0. 00000100:00000001:1.0:1775776236.859945:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776236.859953:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307795718 00000100:00000010:1.0:1775776236.859956:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e29c0. 00000100:00000001:1.0:1775776236.859959:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776236.859962:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776236.859965:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776236.859967:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776236.859972:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776236/real 1775776236] req@ffff8f6202c71c00 x1862022710399232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776236.859982:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776236.859984:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776236.859987:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776236.859990:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776236.859991:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776236.859994:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776236.859999:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71c00 x1862022710399232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776236.860008:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71c00 x1862022710399232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.860015:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776236.860016:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776236.860036:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776236.860039:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776236.860048:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71c00 x1862022710399232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.860056:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776236.860061:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71c00 x1862022710399232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.860068:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776236.860072:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776236.860073:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776236.860075:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776236.860078:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776236.860080:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776236.860083:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776236.860087:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776236.860089:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776236.860093:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71c00 x1862022710399232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776236.860103:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710399232:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776236.860107:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776236.860109:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776236.860112:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71c00 x1862022710399232/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.860119:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776236.860122:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776236.860125:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776236.860129:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776236.860130:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776236.860132:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776236.860155:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776236.860158:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37400. 02000000:00000001:1.0:1775776236.860162:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776236.860164:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776236.860166:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71c00. 00000100:00000001:1.0:1775776236.860170:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776236.860171:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776236.860173:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776236.860180:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776236.860181:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776236.860183:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776236.860185:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776236.860186:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776236.860191:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73480 x1862022710399360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.860199:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776236.860201:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776236.860203:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776236.860205:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776236.860207:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776236.860210:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710399360:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776236.860214:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776236.860217:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776236.860218:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776236.860220:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776236.860223:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776236.860225:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776236.860230:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776236.860233:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776236.860237:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776236.860241:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710399360, portal 10 00000100:00000001:1.0:1775776236.860243:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776236.860244:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849017472 : -123823860534144 : ffff8f6202c73480) 00000100:00000040:1.0:1775776236.860250:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73480 x1862022710399360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776261 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.860257:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776236.860260:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776236.860262:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776236.860265:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710399360, offset 0 00000400:00000200:1.0:1775776236.860269:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776236.860274:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776236.860278:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776236.860281:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776236.860283:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776236.860286:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776236.860289:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776236.860292:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453672:453672:256:4294967295] 192.168.201.115@tcp LPNI seq info [453597:453597:8:4294967295] 00000400:00000200:1.0:1775776236.860297:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776236.860303:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776236.860308:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776236.860312:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776236.860317:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776236.860320:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775776236.860324:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776236.860327:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1775776236.860331:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776236.860333:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776236.860336:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776236.860340:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776236.860343:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776236.860345:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776236.860348:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776236.860352:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73480 x1862022710399360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776261 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.860361:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776236.860363:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73480 x1862022710399360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776261 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.860371:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776236.860372:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776236.860376:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776236.860380:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776236.860384:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776236.860386:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307791884 00000100:00000001:1.0:1775776236.860389:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307795718 00000100:00000001:1.0:1775776236.860392:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307791884 00000100:00000001:1.0:1775776236.860394:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307795718 00000100:00000001:1.0:1775776236.860396:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776236.860398:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776236.860400:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776236.860404:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776236/real 1775776236] req@ffff8f6202c73480 x1862022710399360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776236.860413:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776236.860414:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776236.860416:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776236.860418:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776236.860420:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776236.860423:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776236.860426:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73480 x1862022710399360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776236.860435:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73480 x1862022710399360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.860442:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776236.860443:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776236.860445:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776236.860447:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776236.860450:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73480 x1862022710399360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.860457:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776236.860461:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73480 x1862022710399360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.860467:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776236.860470:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776236.860472:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776236.860473:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776236.860475:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776236.860480:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776236.860483:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776236.860487:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776236.860489:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776236.860493:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73480 x1862022710399360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776236.860500:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710399360:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776236.860504:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776236.860507:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70380 x1862022710399488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.860513:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776236.860515:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776236.860517:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776236.860519:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776236.860520:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776236.860523:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710399488:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776236.860527:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776236.860529:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776236.860530:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776236.860532:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776236.860536:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776236.860538:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776236.860541:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776236.860543:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776236.860547:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776236.860550:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710399488, portal 10 00000100:00000001:1.0:1775776236.860552:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776236.860553:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004928 : -123823860546688 : ffff8f6202c70380) 00000100:00000040:1.0:1775776236.860559:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70380 x1862022710399488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776261 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.860565:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776236.860568:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776236.860571:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776236.860573:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710399488, offset 0 00000400:00000200:1.0:1775776236.860577:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776236.860580:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776236.860584:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776236.860587:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776236.860590:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776236.860592:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776236.860594:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776236.860597:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453673:453673:256:4294967295] 192.168.201.115@tcp LPNI seq info [453598:453598:8:4294967295] 00000400:00000200:1.0:1775776236.860601:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776236.860607:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776236.860612:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776236.860615:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776236.860619:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776236.860622:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775776236.860625:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776236.860628:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1775776236.860632:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776236.860634:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776236.860637:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776236.860641:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776236.860644:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776236.860647:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776236.860650:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776236.860653:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70380 x1862022710399488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776261 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.860661:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776236.860664:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70380 x1862022710399488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776261 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.860671:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776236.860673:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776236.860677:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776236.860680:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776236.860684:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776236.860686:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307791884 00000100:00000001:1.0:1775776236.860688:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307795718 00000100:00000001:1.0:1775776236.860691:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307791884 00000100:00000001:1.0:1775776236.860693:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307795718 00000100:00000001:1.0:1775776236.860695:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776236.860697:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776236.860699:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776236.860703:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776236/real 1775776236] req@ffff8f6202c70380 x1862022710399488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776236.860712:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776236.860713:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776236.860715:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776236.860718:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776236.860719:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776236.860722:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776236.860726:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70380 x1862022710399488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776236.860735:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70380 x1862022710399488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.860741:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776236.860743:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776236.860744:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776236.860746:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776236.860749:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70380 x1862022710399488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.860756:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776236.860760:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70380 x1862022710399488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.860768:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776236.860770:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776236.860772:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776236.860774:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776236.860775:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776236.860777:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776236.860780:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776236.860783:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776236.860785:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776236.860789:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70380 x1862022710399488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776236.860797:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710399488:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776236.860802:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776236.860804:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776236.860807:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73480 x1862022710399360/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.860814:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776236.860815:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776236.860817:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776236.860820:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776236.860822:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776236.860823:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776236.860825:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776236.860827:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35400. 02000000:00000001:1.0:1775776236.860830:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776236.860831:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776236.860833:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73480. 00000100:00000001:1.0:1775776236.860836:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776236.860838:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776236.860839:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776236.860843:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70380 x1862022710399488/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776261 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776236.860850:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776236.860851:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776236.860852:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776236.860855:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776236.860856:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776236.860857:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776236.860859:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776236.860860:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35800. 02000000:00000001:1.0:1775776236.860863:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776236.860864:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776236.860866:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70380. 00000100:00000001:1.0:1775776236.860868:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776236.860869:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776236.860871:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776236.860876:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776236.860877:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776236.860879:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776236.860881:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776236.987191:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776236.987196:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776236.987200:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776236.987201:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776236.987203:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776236.987204:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776236.987260:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776236.987263:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776236.987266:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776237.883246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776237.883252:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776237.883254:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776237.883259:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776237.883262:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776237.883264:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776237.883266:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776237.883268:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776237.883270:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776238.011271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776238.011277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776238.011282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776238.011284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776238.011288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776238.011289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776238.011358:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776238.011362:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776238.011366:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776238.907254:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776238.907260:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776238.907263:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776238.907269:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776238.907273:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776238.907277:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776238.907288:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776238.907291:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776238.907294:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776239.035254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776239.035259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776239.035263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776239.035264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776239.035267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776239.035268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776239.035347:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776239.035351:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776239.035354:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776239.931264:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776239.931272:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776239.931274:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776239.931280:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776239.931284:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776239.931288:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776239.931291:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776239.931293:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776239.931296:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776240.059267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776240.059273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776240.059279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776240.059282:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776240.059283:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776240.059286:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776240.059289:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776240.059292:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776240.059294:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776240.059296:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776240.059301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776240.059302:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776240.059304:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776240.059305:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776240.059307:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776240.059308:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776240.059310:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776240.059312:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776240.059382:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776240.059387:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776240.059390:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776240.763237:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307795718, at 4307795776 00000100:00000040:0.0:1775776240.763247:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307795718, at 4307795776 00000100:00000010:1.0:1775776240.763253:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e29c0. 00000100:00000010:0.0:1775776240.763258:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e22a0. 00000100:00000001:1.0:1775776240.955258:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776240.955264:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776240.955265:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776240.955269:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776240.955272:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776240.955275:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776240.955278:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776240.955280:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776240.955282:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776241.083252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776241.083262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776241.083268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776241.083270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776241.083272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776241.083273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776241.083340:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776241.083343:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776241.083346:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776241.979242:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775776241.979244:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776241.979249:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776241.979251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776241.979254:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1775776241.979258:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776241.979258:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776241.979264:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776241.979264:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776241.979270:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776241.979271:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776241.979275:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776241.979279:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775776241.979280:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13124 00000100:00000001:1.0:1775776241.979285:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776241.979292:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776241.979300:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776241.979304:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776241.979308:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776241.979323:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776241.979326:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776241.979329:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776241.979333:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776241.979335:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776241.979338:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776241.979341:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776241.979343:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776241.979346:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776241.979349:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776241.979352:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776241.979354:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776241.979356:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776241.979360:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776241.979363:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776241.979366:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776241.979369:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776241.979373:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776241.979375:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776241.979378:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776241.979380:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776241.979393:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775776241.979396:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776241.979400:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776241.979402:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776241.979405:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776241.979408:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776241.979412:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776241.979424:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:3.0:1775776241.979431:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776241.979441:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710399616/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776241.979452:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776241.979457:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776241.979462:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710399616/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776241.979485:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776241.979488:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776241.979492:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776241.979494:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776241.979496:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776241.979499:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776241.979502:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13124 00000400:00000200:3.0:1775776241.979507:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776241.979509:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776241.979512:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1775776241.979513:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:1.0:1775776241.979514:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1775776241.979518:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776241.979523:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71500 x1862022710399616/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776241.979525:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776241.979530:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776241.979536:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776241.979539:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776241.979541:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776241.979541:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776241.979546:0:191261:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000001:1.0:1775776241.979547:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:3.0:1775776241.979549:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 02000000:00000001:1.0:1775776241.979550:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775776241.979552:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:1.0:1775776241.979555:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776241.979556:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00100000:1.0:1775776241.979559:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710399616:192.168.201.115@tcp:250:kworker.0 00000020:00000001:3.0:1775776241.979561:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776241.979565:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1775776241.979567:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776241.979576:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1775776241.979580:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776241.979583:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775776241.979584:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776241.979586:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776241.979588:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775776241.979592:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776241.979594:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000010:1.0:1775776241.979596:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 00000100:00080000:3.0:1775776241.979599:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000001:1.0:1775776241.979600:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776241.979604:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776241.979608:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776241.979611:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776241.979615:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1775776241.979616:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 02000000:00000001:3.0:1775776241.979621:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776241.979625:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000010:1.0:1775776241.979626:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000008:00100000:3.0:1775776241.979630:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776241.979634:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776241.979635:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710399616, portal 25 00000100:00000001:3.0:1775776241.979637:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776241.979639:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776241.979641:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849009408 : -123823860542208 : ffff8f6202c71500) 00000100:00000010:3.0:1775776241.979643:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000100:00000040:1.0:1775776241.979648:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71500 x1862022710399616/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776266 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775776241.979649:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776241.979656:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776241.979659:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776241.979661:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775776241.979663:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:1.0:1775776241.979666:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 02000000:00000001:3.0:1775776241.979667:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1775776241.979671:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 02000000:00000001:3.0:1775776241.979671:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776241.979676:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710399616, offset 0 02000000:00000010:3.0:1775776241.979679:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000100:00000001:3.0:1775776241.979685:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776241.979686:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00100000:3.0:1775776241.979695:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710399744/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776241.979699:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776241.979710:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:3.0:1775776241.979710:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776241.979716:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776241.979720:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1775776241.979722:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710399744/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776241.979728:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776241.979735:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1775776241.979740:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776241.979741:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00080000:3.0:1775776241.979746:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776241.979747:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453674:453674:256:4294967295] 192.168.201.115@tcp LPNI seq info [453599:453599:8:4294967295] 00000100:00080000:3.0:1775776241.979755:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:1.0:1775776241.979758:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776241.979760:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776241.979763:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776241.979769:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775776241.979772:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00080000:3.0:1775776241.979776:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13124 00000800:00000200:1.0:1775776241.979782:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000400:00000200:3.0:1775776241.979786:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000010:1.0:1775776241.979788:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000400:00000200:3.0:1775776241.979793:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776241.979796:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1775776241.979798:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776241.979803:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000100:00000040:3.0:1775776241.979804:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776241.979809:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1775776241.979812:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776241.979815:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:1.1:1775776241.979817:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000100:00000001:3.0:1775776241.979821:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000100:1.0:1775776241.979825:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000020:00000001:3.0:1775776241.979827:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000800:00000010:1.0:1775776241.979828:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000020:00000040:3.0:1775776241.979830:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000400:00000200:1.0:1775776241.979834:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:3.0:1775776241.979834:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776241.979837:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000400:00000200:1.0:1775776241.979841:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000020:00000001:3.0:1775776241.979843:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776241.979845:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000020:00000001:3.0:1775776241.979847:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000010:1.0:1775776241.979848:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000040:3.0:1775776241.979855:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776241.979856:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1775776241.979860:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1775776241.979862:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71500 x1862022710399616/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776266 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776241.979866:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776241.979872:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:1.0:1775776241.979877:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000040:3.0:1775776241.979878:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00000040:1.0:1775776241.979882:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71500 x1862022710399616/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776266 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775776241.979886:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776241.979893:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776241.979898:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776241.979899:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776241.979901:0:2411:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1775776241.979903:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776241.979906:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776241.979908:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776241.979911:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776241.979914:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776241.979915:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776241.979919:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776241.979923:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776241.979926:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307800838 00000100:00000010:3.0:1775776241.979926:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775776241.979948:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000100:00000010:1.0:1775776241.979951:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e29c0. 02000000:00000001:3.0:1775776241.979951:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776241.979954:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776241.979958:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776241.979959:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775776241.979960:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1775776241.979963:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776241.979967:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307800838 02000000:00000010:3.0:1775776241.979970:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000010:1.0:1775776241.979971:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e21e0. 00000100:00000001:1.0:1775776241.979975:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775776241.979976:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776241.979978:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776241.979981:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776241.979984:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710399872/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776241.979985:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776241.979991:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776241/real 1775776241] req@ffff8f6202c71500 x1862022710399616/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776241.980006:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000001:1.0:1775776241.980009:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1775776241.980010:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1775776241.980011:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776241.980015:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776241.980017:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000100:00000040:3.0:1775776241.980018:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710399872/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1775776241.980019:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776241.980023:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776241.980029:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71500 x1862022710399616/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776241.980038:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776241.980044:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1775776241.980045:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71500 x1862022710399616/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776241.980051:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776241.980058:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776241.980064:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:3.0:1775776241.980064:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776241.980068:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776241.980071:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13134) 00000100:00000001:1.0:1775776241.980072:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776241.980076:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776241.980082:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71500 x1862022710399616/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776241.980090:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776241.980096:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71500 x1862022710399616/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776241.980103:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776241.980105:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776241.980106:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776241.980108:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776241.980110:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776241.980112:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776241.980114:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776241.980117:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776241.980118:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776241.980121:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71500 x1862022710399616/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776241.980127:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710399616:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776241.980130:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776241.980131:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776241.980149:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71500 x1862022710399616/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776241.980154:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776241.980156:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776241.980157:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776241.980160:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776241.980161:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776241.980162:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776241.980163:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776241.980165:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37c00. 02000000:00000001:1.0:1775776241.980168:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776241.980170:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776241.980171:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71500. 00000100:00000001:1.0:1775776241.980173:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776241.980174:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776241.980175:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776241.980181:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776241.980182:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776241.980183:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776241.980184:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776241.980185:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776241.980188:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70a80 x1862022710399744/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776241.980193:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776241.980194:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776241.980196:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776241.980197:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776241.980198:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776241.980200:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710399744:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776241.980203:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776241.980205:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776241.980206:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776241.980207:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776241.980209:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776241.980211:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776241.980214:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776241.980216:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776241.980219:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776241.980221:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710399744, portal 10 00000100:00000001:1.0:1775776241.980222:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776241.980223:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849006720 : -123823860544896 : ffff8f6202c70a80) 00000100:00000040:1.0:1775776241.980228:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70a80 x1862022710399744/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776266 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776241.980233:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776241.980235:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776241.980236:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776241.980238:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710399744, offset 0 00000400:00000200:1.0:1775776241.980240:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776241.980243:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776241.980245:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776241.980248:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776241.980250:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776241.980252:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776241.980254:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776241.980258:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453675:453675:256:4294967295] 192.168.201.115@tcp LPNI seq info [453600:453600:8:4294967295] 00000400:00000200:1.0:1775776241.980263:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776241.980269:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776241.980274:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776241.980278:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776241.980282:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776241.980285:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775776241.980289:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776241.980292:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1775776241.980297:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776241.980299:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776241.980302:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776241.980305:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776241.980307:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776241.980309:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776241.980311:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776241.980313:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70a80 x1862022710399744/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776266 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776241.980319:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776241.980321:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70a80 x1862022710399744/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776266 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776241.980333:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776241.980334:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776241.980336:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776241.980339:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776241.980342:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776241.980343:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307797004 00000100:00000001:1.0:1775776241.980345:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307800838 00000100:00000001:1.0:1775776241.980347:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307797004 00000100:00000001:1.0:1775776241.980348:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307800838 00000100:00000001:1.0:1775776241.980349:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776241.980352:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776241.980353:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776241.980355:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776241/real 1775776241] req@ffff8f6202c70a80 x1862022710399744/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776241.980361:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776241.980362:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776241.980363:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776241.980365:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776241.980366:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776241.980367:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776241.980369:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70a80 x1862022710399744/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776241.980374:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70a80 x1862022710399744/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776241.980378:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776241.980379:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776241.980380:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776241.980381:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776241.980383:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70a80 x1862022710399744/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776241.980387:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776241.980389:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70a80 x1862022710399744/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776241.980394:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776241.980395:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776241.980396:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776241.980398:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776241.980399:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776241.980403:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776241.980404:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776241.980407:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776241.980408:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776241.980411:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70a80 x1862022710399744/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776241.980416:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710399744:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776241.980418:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776241.980420:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70700 x1862022710399872/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776241.980424:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776241.980425:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776241.980426:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776241.980427:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776241.980429:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776241.980431:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710399872:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776241.980433:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776241.980435:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776241.980436:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776241.980437:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776241.980439:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776241.980441:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776241.980443:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776241.980445:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776241.980447:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776241.980449:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710399872, portal 10 00000100:00000001:1.0:1775776241.980450:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776241.980451:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849005824 : -123823860545792 : ffff8f6202c70700) 00000100:00000040:1.0:1775776241.980455:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70700 x1862022710399872/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776266 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776241.980460:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776241.980462:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776241.980463:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776241.980464:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710399872, offset 0 00000400:00000200:1.0:1775776241.980467:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776241.980470:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776241.980472:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776241.980474:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776241.980476:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776241.980478:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776241.980479:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776241.980481:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453676:453676:256:4294967295] 192.168.201.115@tcp LPNI seq info [453601:453601:8:4294967295] 00000400:00000200:1.0:1775776241.980484:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776241.980489:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776241.980492:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776241.980494:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776241.980496:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776241.980498:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775776241.980501:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776241.980503:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1775776241.980506:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776241.980507:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776241.980509:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776241.980511:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776241.980513:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:1.0:1775776241.980514:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:1.0:1775776241.980516:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776241.980518:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70700 x1862022710399872/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776266 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776241.980523:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776241.980525:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70700 x1862022710399872/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776266 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776241.980529:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776241.980531:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776241.980532:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776241.980534:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776241.980536:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776241.980537:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307797004 00000100:00000001:1.0:1775776241.980539:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307800838 00000100:00000001:1.0:1775776241.980540:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307797004 00000100:00000001:1.0:1775776241.980542:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307800838 00000100:00000001:1.0:1775776241.980543:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776241.980544:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776241.980545:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776241.980547:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776241/real 1775776241] req@ffff8f6202c70700 x1862022710399872/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776241.980552:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776241.980553:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776241.980554:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776241.980555:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776241.980556:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776241.980557:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776241.980559:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70700 x1862022710399872/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776241.980564:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70700 x1862022710399872/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776241.980569:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776241.980570:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776241.980571:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776241.980573:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776241.980576:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70700 x1862022710399872/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776241.980583:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776241.980587:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70700 x1862022710399872/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776241.980594:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776241.980596:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776241.980598:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776241.980599:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776241.980601:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776241.980603:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776241.980605:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776241.980609:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776241.980611:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776241.980616:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70700 x1862022710399872/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776241.980625:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710399872:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776241.980629:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776241.980631:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776241.980633:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70a80 x1862022710399744/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776241.980638:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776241.980639:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776241.980641:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776241.980643:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776241.980644:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776241.980645:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776241.980646:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776241.980647:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35000. 02000000:00000001:1.0:1775776241.980649:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776241.980650:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776241.980651:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70a80. 00000100:00000001:1.0:1775776241.980652:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776241.980654:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776241.980655:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776241.980657:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70700 x1862022710399872/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776266 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776241.980662:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776241.980662:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776241.980663:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776241.980665:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776241.980665:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776241.980666:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776241.980667:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776241.980668:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37800. 02000000:00000001:1.0:1775776241.980669:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776241.980670:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776241.980670:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70700. 00000100:00000001:1.0:1775776241.980671:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776241.980672:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776241.980673:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776241.980676:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776241.980676:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776241.980678:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776241.980679:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776242.107250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776242.107255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776242.107259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776242.107260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776242.107263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776242.107264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776242.107329:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776242.107332:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776242.107334:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776243.003273:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776243.003278:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776243.003279:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776243.003284:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776243.003287:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776243.003294:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776243.003297:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776243.003299:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776243.003301:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776243.131205:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776243.131211:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776243.131215:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776243.131216:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776243.131218:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776243.131220:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776243.131273:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776243.131278:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776243.131282:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776244.027245:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776244.027251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776244.027263:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776244.027270:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776244.027273:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776244.027277:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776244.027281:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776244.027282:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776244.027284:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776244.155202:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776244.155206:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776244.155210:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776244.155211:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776244.155213:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776244.155214:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776244.155271:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776244.155276:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776244.155279:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776245.051235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776245.051241:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776245.051242:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776245.051246:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776245.051249:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776245.051252:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776245.051253:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776245.051256:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776245.051258:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776245.179246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776245.179256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776245.179262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776245.179263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776245.179266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776245.179267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776245.179334:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776245.179338:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776245.179341:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776245.883215:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307800838, at 4307800896 00000100:00000040:0.0:1775776245.883228:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307800838, at 4307800896 00000100:00000010:1.0:1775776245.883228:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e21e0. 00000100:00000010:0.0:1775776245.883235:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e29c0. 00000100:00000001:1.0:1775776246.075241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776246.075248:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776246.075249:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776246.075255:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776246.075258:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776246.075262:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776246.075265:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776246.075267:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776246.075270:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776246.203246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776246.203252:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776246.203254:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776246.203257:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776246.203260:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776246.203262:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776246.203263:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776246.203264:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776246.203267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776246.203268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776246.203271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776246.203272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776246.203340:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776246.203344:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776246.203347:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776247.099227:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776247.099232:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776247.099234:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776247.099239:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:0.0:1775776247.099241:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776247.099242:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776247.099246:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775776247.099247:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776247.099250:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776247.099250:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776247.099253:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775776247.099254:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776247.099257:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776247.099260:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776247.099268:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13129 00000400:00000200:0.0:1775776247.099278:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776247.099284:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776247.099287:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776247.099291:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776247.099293:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776247.099296:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776247.099298:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776247.099302:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776247.099304:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776247.099306:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776247.099309:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776247.099311:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776247.099313:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776247.099316:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776247.099318:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776247.099320:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776247.099322:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776247.099325:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776247.099328:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776247.099330:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776247.099334:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776247.099336:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776247.099340:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776247.099343:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776247.099345:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776247.099354:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060700. 00000020:00000040:0.0:1775776247.099357:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776247.099361:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776247.099363:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776247.099365:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776247.099368:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776247.099372:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776247.099380:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4800. 00000100:00000001:0.0:1775776247.099386:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776247.099394:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060700 x1862022710400000/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776247.099404:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776247.099408:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776247.099412:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000225c8695] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060700 x1862022710400000/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776247.099432:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776247.099435:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776247.099438:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776247.099440:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776247.099441:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776247.099444:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776247.099447:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13129 00000400:00000200:0.0:1775776247.099452:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776247.099456:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:1.0:1775776247.099457:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1775776247.099459:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776247.099461:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1775776247.099463:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776247.099464:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1775776247.099467:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776247.099470:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000040:1.0:1775776247.099474:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e060700 x1862022710400000/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776247.099475:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776247.099479:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776247.099481:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776247.099484:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776247.099487:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:1.0:1775776247.099490:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1775776247.099491:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776247.099493:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776247.099495:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776247.099498:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1775776247.099500:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776247.099502:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776247.099503:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776247.099507:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776247.099508:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776247.099510:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00100000:1.0:1775776247.099514:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e060700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710400000:192.168.201.115@tcp:250:kworker.0 00000020:00000040:0.0:1775776247.099515:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775776247.099522:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776247.099523:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1775776247.099525:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776247.099527:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776247.099528:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776247.099530:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776247.099533:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1775776247.099534:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776247.099538:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776247.099538:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 00000100:00000001:0.0:1775776247.099541:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776247.099542:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1775776247.099547:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776247.099553:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776247.099553:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776247.099557:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776247.099559:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000100:00000010:0.0:1775776247.099561:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061880. 00000020:00000040:0.0:1775776247.099565:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776247.099567:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776247.099568:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776247.099568:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776247.099576:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710400000, portal 25 00000100:00000001:0.0:1775776247.099578:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775776247.099580:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1775776247.099584:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775776247.099584:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574554880 : -123823134996736 : ffff8f622e060700) 02000000:00000001:0.0:1775776247.099587:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776247.099592:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5c00. 00000100:00000001:0.0:1775776247.099596:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776247.099601:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e060700 x1862022710400000/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776247.099602:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061880 x1862022710400128/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776247.099630:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1775776247.099638:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1775776247.099638:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:0.0:1775776247.099642:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776247.099644:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000040:0.0:1775776247.099648:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005afd1f27] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061880 x1862022710400128/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776247.099649:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710400000, offset 0 00000100:00000001:0.0:1775776247.099660:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776247.099661:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00080000:0.0:1775776247.099664:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776247.099667:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776247.099669:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776247.099671:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775776247.099672:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776247.099677:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776247.099682:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13129 00000400:00000200:1.0:1775776247.099682:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776247.099690:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776247.099695:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776247.099698:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776247.099700:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776247.099704:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453677:453677:256:4294967295] 192.168.201.115@tcp LPNI seq info [453602:453602:8:4294967295] 00000400:00000200:1.0:1775776247.099707:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776247.099716:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000400:00000200:0.0:1775776247.099723:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000200:1.0:1775776247.099724:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776247.099730:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000800:00000010:1.0:1775776247.099731:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000100:00080000:0.0:1775776247.099734:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776247.099739:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776247.099743:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.0:1775776247.099743:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:0.0:1775776247.099748:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:1.1:1775776247.099751:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775776247.099752:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776247.099757:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776247.099758:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776247.099759:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776247.099761:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775776247.099763:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775776247.099765:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775776247.099769:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776247.099770:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776247.099773:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776247.099777:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776247.099778:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776247.099782:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:1.0:1775776247.099783:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307805958 00000020:00000040:0.0:1775776247.099786:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775776247.099790:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000010:1.0:1775776247.099791:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e21e0. 00000100:00000001:0.0:1775776247.099792:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776247.099795:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776247.099798:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776247.099799:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1775776247.099802:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776247.099804:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776247.099806:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775776247.099807:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307805958 00000008:00000001:0.0:1775776247.099810:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776247.099813:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776247.099813:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2cc0. 00000100:00000001:1.0:1775776247.099819:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000010:0.0:1775776247.099820:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061c00. 00000020:00000040:0.0:1775776247.099824:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000100:00000001:1.0:1775776247.099824:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776247.099828:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1775776247.099829:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776247.099831:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776247.099833:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776247.099834:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776247.099837:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775776247.099837:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1775776247.099840:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776247.099857:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7000. 00000100:00000001:1.0:1775776247.099857:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776247.099860:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1775776247.099863:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776247.099863:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776247.099866:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776247.099868:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00100000:0.0:1775776247.099871:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061c00 x1862022710400256/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1775776247.099877:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061880 x1862022710400128/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776247.099888:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776247.099891:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776247.099895:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1775776247.099898:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076a44879] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061c00 x1862022710400256/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776247.099898:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776247.099903:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776247.099906:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776247.099908:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776247.099911:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776247.099912:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776247.099916:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776247.099919:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776247.099922:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776247.099926:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13139) 00000100:00100000:1.0:1775776247.099926:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710400128:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776247.099931:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776247.099933:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776247.099934:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776247.099936:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776247.099940:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776247.099942:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776247.099946:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776247.099949:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fe000. 00000400:00000010:1.0:1775776247.099953:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776247.099957:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710400128, portal 10 00000100:00000001:1.0:1775776247.099959:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776247.099960:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574559360 : -123823134992256 : ffff8f622e061880) 00000100:00000040:1.0:1775776247.099967:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061880 x1862022710400128/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776247.099974:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776247.099976:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776247.099978:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776247.099980:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710400128, offset 0 00000400:00000200:1.0:1775776247.099983:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776247.099987:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776247.099990:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776247.099992:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776247.099995:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776247.099997:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776247.099999:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776247.100002:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453678:453678:255:4294967295] 192.168.201.115@tcp LPNI seq info [453603:453603:7:4294967295] 00000400:00000200:1.0:1775776247.100005:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776247.100011:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776247.100016:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776247.100019:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230400. 00000800:00000200:1.0:1775776247.100022:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776247.100025:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:1.0:1775776247.100026:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776247.100029:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776247.100037:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776247.100039:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307802123 00000100:00000001:1.0:1775776247.100040:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307805958 00000100:00000001:1.0:1775776247.100044:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307802123 00000100:00000001:1.0:1775776247.100046:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307805958 00000100:00000001:1.0:1775776247.100048:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776247.100050:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776247.100052:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776247.100054:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776247.100057:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776247.100058:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775776247.100059:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776247.100061:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776247.100062:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776247.100066:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061c00 x1862022710400256/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776247.100072:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776247.100073:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776247.100075:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776247.100077:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776247.100078:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776247.100081:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710400256:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776247.100084:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776247.100086:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776247.100087:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776247.100088:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776247.100091:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776247.100093:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776247.100096:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776247.100098:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776247.100101:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776247.100105:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710400256, portal 10 00000100:00000001:1.0:1775776247.100107:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776247.100108:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574560256 : -123823134991360 : ffff8f622e061c00) 00000100:00000040:1.0:1775776247.100114:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061c00 x1862022710400256/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776247.100120:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776247.100122:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776247.100124:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f33b8. 00000100:00000200:1.0:1775776247.100127:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710400256, offset 0 00000400:00000200:1.0:1775776247.100130:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776247.100154:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776247.100157:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776247.100159:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776247.100162:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776247.100164:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776247.100166:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776247.100169:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453679:453679:254:4294967295] 192.168.201.115@tcp LPNI seq info [453604:453604:6:4294967295] 00000400:00000200:1.0:1775776247.100172:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776247.100178:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776247.100183:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776247.100186:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230600. 00000800:00000200:1.0:1775776247.100189:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776247.100191:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:1.0:1775776247.100192:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776247.100195:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776247.100197:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776247.100198:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307802124 00000100:00000001:1.0:1775776247.100200:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307805958 00000100:00000001:1.0:1775776247.100201:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307802124 00000100:00000001:1.0:1775776247.100203:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307805958 00000100:00000001:1.0:1775776247.100204:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776247.100205:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776247.100207:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776247.100208:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776247.100209:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776247.100210:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775776247.100211:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776247.100212:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776247.100213:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776247.100214:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776247.227217:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776247.227222:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776247.227226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776247.227227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776247.227231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776247.227232:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776247.227280:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776247.227284:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776247.227287:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776248.123191:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776248.123196:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776248.123198:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776248.123201:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776248.123204:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776248.123206:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776248.123208:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776248.123210:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776248.123212:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776248.251265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776248.251270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776248.251276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776248.251277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776248.251279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776248.251280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776248.251367:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776248.251370:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776248.251372:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776249.147260:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776249.147267:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776249.147269:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776249.147275:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776249.147291:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776249.147295:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776249.147298:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776249.147301:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776249.147304:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776249.275225:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776249.275231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776249.275235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776249.275236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776249.275238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776249.275239:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776249.275293:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776249.275296:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776249.275299:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776250.171254:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776250.171261:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776250.171262:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776250.171268:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776250.171271:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776250.171275:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776250.171278:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776250.171281:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776250.171284:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:3.0:1775776250.171440:0:2401:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:3.0:1775776250.171480:0:2401:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1775776250.171491:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776250.171495:0:2401:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:3.0:1775776250.171500:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776250.171504:0:2401:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:3.0:1775776250.171509:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775776250.171513:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:3.0:1775776250.171518:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775776250.171522:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775776250.171529:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:3.0:1775776250.171531:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:3.0:1775776250.171538:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776250.171546:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f622e060700 x1862022710400000/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776250.172128:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776250.172183:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e060700 x1862022710400000/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776250.172198:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776250.172200:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775776250.172211:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775776250.172214:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230400. 00000400:00000200:3.0:1775776250.172217:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775776250.172222:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775776250.172226:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:3.0:1775776250.172228:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:3.0:1775776250.172231:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776250.172237:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f622e061880 x1862022710400128/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776250.172244:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776250.172247:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061880 x1862022710400128/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776250.172254:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776250.172255:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775776250.172260:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775776250.172262:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230600. 00000400:00000200:3.0:1775776250.172264:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775776250.172267:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775776250.172270:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f33b8 00000400:00000010:3.0:1775776250.172272:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f33b8. 00000100:00000001:3.0:1775776250.172274:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776250.172279:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f622e061c00 x1862022710400256/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776250.172286:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776250.172289:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061c00 x1862022710400256/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776250.172295:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776250.172297:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776250.172373:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776250.172376:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776250.172378:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776250.172383:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776247/real 1775776250] req@ffff8f622e061c00 x1862022710400256/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776250.172394:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776250.172396:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776250.172399:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776250.172402:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776250.172404:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776250.172407:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776250.172411:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061c00 x1862022710400256/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776250.172418:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061c00 x1862022710400256/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776250.172424:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776250.172426:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776250.172429:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776250.172430:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776250.172433:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061c00 x1862022710400256/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776250.172440:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776250.172444:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061c00 x1862022710400256/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776250.172450:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776250.172452:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776250.172453:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776250.172455:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776250.172457:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776250.172460:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776250.172463:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776250.172466:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776250.172469:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776250.172472:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061c00 x1862022710400256/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776250.172479:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710400256:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776250.172482:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776250.172485:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776247/real 1775776250] req@ffff8f622e061880 x1862022710400128/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776250.172491:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776250.172492:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776250.172494:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fe000. 00000400:00000200:1.0:1775776250.172496:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776250.172497:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776250.172499:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776250.172502:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061880 x1862022710400128/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776250.172509:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061880 x1862022710400128/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776250.172514:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776250.172515:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776250.172516:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776250.172517:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776250.172520:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061880 x1862022710400128/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776250.172530:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776250.172533:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061880 x1862022710400128/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776250.172538:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776250.172540:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776250.172541:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776250.172542:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776250.172543:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776250.172549:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776250.172551:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776250.172553:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776250.172555:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776250.172558:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061880 x1862022710400128/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776250.172565:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710400128:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776250.172569:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776250.172573:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776247/real 1775776250] req@ffff8f622e060700 x1862022710400000/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776250.172580:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776250.172581:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776250.172583:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776250.172585:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776250.172587:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776250.172589:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776250.172592:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e060700 x1862022710400000/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776250.172605:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e060700 x1862022710400000/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776250.172610:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776250.172624:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776250.172626:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776250.172627:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776250.172631:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e060700 x1862022710400000/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776250.172636:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776250.172639:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e060700 x1862022710400000/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776250.172645:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776250.172647:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776250.172648:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776250.172649:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776250.172651:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776250.172653:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776250.172655:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776250.172657:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776250.172659:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776250.172663:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e060700 x1862022710400000/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776250.172669:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e060700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710400000:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776250.172673:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776250.172675:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776250.172677:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061c00 x1862022710400256/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776250.172688:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776250.172691:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776250.172695:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776250.172699:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776250.172702:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776250.172703:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776250.172705:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776250.172708:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e7000. 02000000:00000001:1.0:1775776250.172712:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776250.172714:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776250.172716:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061c00. 00000100:00000001:1.0:1775776250.172719:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776250.172720:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776250.172722:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776250.172724:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061880 x1862022710400128/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776250.172730:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776250.172731:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776250.172733:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776250.172735:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776250.172736:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776250.172737:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776250.172738:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776250.172739:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e5c00. 02000000:00000001:1.0:1775776250.172741:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776250.172742:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776250.172744:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061880. 00000100:00000001:1.0:1775776250.172745:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776250.172746:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776250.172748:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776250.172750:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e060700 x1862022710400000/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776250.172755:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776250.172756:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776250.172758:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776250.172760:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776250.172761:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776250.172762:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776250.172764:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776250.172765:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:1.0:1775776250.172767:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776250.172768:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776250.172769:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e060700. 00000100:00000001:1.0:1775776250.172771:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776250.172772:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776250.172775:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776250.172782:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776250.172783:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776250.172785:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776250.172786:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776250.299262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776250.299268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776250.299273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776250.299276:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776250.299279:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776250.299282:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776250.299284:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776250.299286:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776250.299288:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776250.299290:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776250.299295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776250.299296:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776250.299298:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776250.299299:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776250.299300:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776250.299302:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776250.299303:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776250.299305:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776250.299377:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776250.299381:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776250.299384:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776251.003234:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307805958, at 4307806016 00000100:00000040:1.0:1775776251.003234:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307805958, at 4307806016 00000100:00000010:0.0:1775776251.003248:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e21e0. 00000100:00000010:1.0:1775776251.003255:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2cc0. 00000100:00000001:1.0:1775776251.195217:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776251.195222:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776251.195224:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776251.195228:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776251.195230:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776251.195233:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776251.195235:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776251.195237:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776251.195239:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776251.323196:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776251.323201:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776251.323207:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776251.323208:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776251.323213:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776251.323214:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776251.323265:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776251.323268:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776251.323270:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776252.219243:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776252.219244:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776252.219250:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1775776252.219252:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776252.219253:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776252.219256:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1775776252.219259:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:0.0:1775776252.219260:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776252.219265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776252.219266:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776252.219269:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776252.219273:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775776252.219274:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13134 00000100:00000001:1.0:1775776252.219276:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775776252.219285:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776252.219288:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776252.219292:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776252.219296:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776252.219299:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776252.219302:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776252.219305:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776252.219308:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776252.219312:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776252.219314:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776252.219317:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776252.219319:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776252.219321:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776252.219324:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776252.219327:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776252.219329:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776252.219331:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776252.219332:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776252.219336:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776252.219339:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776252.219342:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776252.219346:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776252.219349:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776252.219351:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776252.219354:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776252.219356:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776252.219362:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063100. 00000020:00000040:0.0:1775776252.219365:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776252.219369:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776252.219370:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776252.219373:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776252.219375:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776252.219379:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776252.219387:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5800. 00000100:00000001:0.0:1775776252.219392:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776252.219400:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063100 x1862022710400384/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776252.219410:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776252.219415:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776252.219420:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033f6db7f] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063100 x1862022710400384/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776252.219450:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776252.219454:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776252.219457:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776252.219459:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776252.219460:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776252.219463:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776252.219465:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13134 00000400:00000200:0.0:1775776252.219469:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776252.219472:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776252.219474:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776252.219477:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776252.219479:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776252.219482:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776252.219483:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776252.219486:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776252.219487:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776252.219489:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776252.219491:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775776252.219493:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775776252.219495:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775776252.219498:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776252.219499:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776252.219501:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776252.219503:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776252.219506:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775776252.219508:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776252.219511:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776252.219512:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776252.219517:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776252.219519:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776252.219521:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776252.219525:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776252.219528:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776252.219530:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776252.219534:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060e00. 00000020:00000040:0.0:1775776252.219536:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776252.219539:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776252.219540:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776252.219541:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776252.219542:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776252.219544:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776252.219548:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7800. 00000100:00000001:0.0:1775776252.219551:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776252.219557:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060e00 x1862022710400512/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776252.219565:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776252.219567:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776252.219571:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004a3bddaa] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060e00 x1862022710400512/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776252.219579:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776252.219581:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776252.219584:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776252.219586:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776252.219587:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776252.219590:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776252.219592:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13134 00000400:00000200:0.0:1775776252.219595:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776252.219598:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776252.219600:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776252.219602:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776252.219604:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776252.219606:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776252.219608:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776252.219610:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776252.219611:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776252.219613:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776252.219615:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775776252.219617:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775776252.219619:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775776252.219622:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776252.219623:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776252.219625:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776252.219627:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776252.219630:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775776252.219632:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776252.219634:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776252.219636:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776252.219638:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776252.219640:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776252.219642:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776252.219644:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776252.219646:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776252.219648:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776252.219651:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060700. 00000020:00000040:0.0:1775776252.219654:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776252.219656:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776252.219657:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776252.219658:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776252.219660:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776252.219662:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776252.219665:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6000. 00000100:00000001:0.0:1775776252.219668:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776252.219673:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060700 x1862022710400640/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776252.219680:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776252.219682:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776252.219686:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000225c8695] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060700 x1862022710400640/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776252.219693:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776252.219696:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776252.219700:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776252.219703:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776252.219706:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776252.219712:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13144) 00000100:00000001:1.0:1775776252.219781:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776252.219784:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776252.219786:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776252.219792:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e063100 x1862022710400384/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.219800:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776252.219802:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776252.219806:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776252.219809:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776252.219811:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776252.219815:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e063100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710400384:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776252.219820:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776252.219823:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776252.219826:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776252.219828:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776252.219834:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776252.219836:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776252.219841:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776252.219847:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776252.219853:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776252.219860:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710400384, portal 25 00000100:00000001:1.0:1775776252.219863:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776252.219865:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574565632 : -123823134985984 : ffff8f622e063100) 00000100:00000040:1.0:1775776252.219870:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e063100 x1862022710400384/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.219877:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776252.219880:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776252.219882:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776252.219885:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710400384, offset 0 00000400:00000200:1.0:1775776252.219892:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776252.219900:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776252.219905:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776252.219911:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776252.219915:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776252.219919:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776252.219922:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776252.219926:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453680:453680:256:4294967295] 192.168.201.115@tcp LPNI seq info [453605:453605:8:4294967295] 00000400:00000200:1.0:1775776252.219931:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776252.219941:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776252.219947:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776252.219953:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776252.219961:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776252.219965:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775776252.219970:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776252.219974:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775776252.219978:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776252.219980:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776252.219985:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776252.219990:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776252.219995:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776252.219997:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776252.220001:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776252.220005:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e063100 x1862022710400384/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220015:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776252.220018:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e063100 x1862022710400384/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220025:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776252.220027:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776252.220033:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776252.220037:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776252.220039:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307811078 00000100:00000010:1.0:1775776252.220046:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2cc0. 00000100:00000001:1.0:1775776252.220052:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776252.220058:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307811078 00000100:00000010:1.0:1775776252.220061:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2d20. 00000100:00000001:1.0:1775776252.220064:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776252.220067:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776252.220070:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776252.220073:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776252.220077:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776252/real 1775776252] req@ffff8f622e063100 x1862022710400384/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776252.220086:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776252.220088:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776252.220091:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776252.220093:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776252.220095:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776252.220097:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776252.220100:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e063100 x1862022710400384/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776252.220110:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e063100 x1862022710400384/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220117:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776252.220118:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776252.220120:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776252.220122:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776252.220125:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e063100 x1862022710400384/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220131:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776252.220156:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e063100 x1862022710400384/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220163:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776252.220166:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776252.220168:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776252.220169:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776252.220172:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776252.220173:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776252.220176:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776252.220180:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776252.220183:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776252.220188:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e063100 x1862022710400384/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776252.220196:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e063100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710400384:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776252.220200:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776252.220204:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e060e00 x1862022710400512/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220211:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776252.220212:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776252.220214:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776252.220216:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776252.220218:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776252.220221:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e060e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710400512:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776252.220224:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776252.220225:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776252.220227:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776252.220228:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776252.220232:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776252.220233:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776252.220237:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776252.220239:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776252.220243:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776252.220246:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710400512, portal 10 00000100:00000001:1.0:1775776252.220248:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776252.220249:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574556672 : -123823134994944 : ffff8f622e060e00) 00000100:00000040:1.0:1775776252.220255:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e060e00 x1862022710400512/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220262:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776252.220265:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776252.220267:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776252.220269:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710400512, offset 0 00000400:00000200:1.0:1775776252.220272:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776252.220276:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776252.220279:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776252.220282:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776252.220285:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776252.220287:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776252.220290:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776252.220293:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453681:453681:256:4294967295] 192.168.201.115@tcp LPNI seq info [453606:453606:8:4294967295] 00000400:00000200:1.0:1775776252.220297:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776252.220303:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776252.220307:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776252.220310:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776252.220314:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776252.220317:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775776252.220321:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776252.220323:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775776252.220327:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776252.220328:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776252.220331:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776252.220334:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776252.220337:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776252.220339:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776252.220341:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776252.220344:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e060e00 x1862022710400512/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220358:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776252.220361:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e060e00 x1862022710400512/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220367:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776252.220369:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776252.220372:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776252.220375:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776252.220379:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776252.220381:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307807244 00000100:00000001:1.0:1775776252.220383:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307811078 00000100:00000001:1.0:1775776252.220385:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307807244 00000100:00000001:1.0:1775776252.220388:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307811078 00000100:00000001:1.0:1775776252.220390:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776252.220391:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776252.220393:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776252.220397:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776252/real 1775776252] req@ffff8f622e060e00 x1862022710400512/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776252.220404:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776252.220406:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776252.220407:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776252.220409:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776252.220411:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776252.220413:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776252.220416:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e060e00 x1862022710400512/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776252.220424:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e060e00 x1862022710400512/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220437:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776252.220438:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776252.220440:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776252.220441:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776252.220444:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e060e00 x1862022710400512/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220452:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776252.220455:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e060e00 x1862022710400512/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220462:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776252.220464:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776252.220466:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776252.220467:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776252.220469:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776252.220474:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776252.220477:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776252.220480:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776252.220482:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776252.220485:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e060e00 x1862022710400512/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776252.220493:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e060e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710400512:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776252.220497:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776252.220501:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e060700 x1862022710400640/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220508:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776252.220509:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776252.220511:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776252.220512:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776252.220515:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776252.220518:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e060700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710400640:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776252.220521:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776252.220523:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776252.220525:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776252.220526:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776252.220530:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776252.220531:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776252.220534:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776252.220537:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776252.220539:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776252.220542:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710400640, portal 10 00000100:00000001:1.0:1775776252.220543:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776252.220544:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574554880 : -123823134996736 : ffff8f622e060700) 00000100:00000040:1.0:1775776252.220550:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e060700 x1862022710400640/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220556:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776252.220559:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776252.220561:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776252.220563:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710400640, offset 0 00000400:00000200:1.0:1775776252.220567:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776252.220571:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776252.220574:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776252.220577:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776252.220580:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776252.220582:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776252.220584:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776252.220587:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453682:453682:256:4294967295] 192.168.201.115@tcp LPNI seq info [453607:453607:8:4294967295] 00000400:00000200:1.0:1775776252.220591:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776252.220597:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776252.220601:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776252.220604:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776252.220608:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776252.220610:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775776252.220614:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776252.220617:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775776252.220620:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776252.220622:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776252.220625:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776252.220628:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776252.220630:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776252.220632:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776252.220634:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776252.220637:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e060700 x1862022710400640/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220645:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776252.220647:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e060700 x1862022710400640/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220653:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776252.220655:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776252.220658:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776252.220661:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776252.220664:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776252.220666:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307807244 00000100:00000001:1.0:1775776252.220668:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307811078 00000100:00000001:1.0:1775776252.220670:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307807244 00000100:00000001:1.0:1775776252.220672:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307811078 00000100:00000001:1.0:1775776252.220674:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776252.220676:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776252.220678:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776252.220681:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776252/real 1775776252] req@ffff8f622e060700 x1862022710400640/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776252.220688:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776252.220690:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776252.220692:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776252.220694:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776252.220695:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776252.220697:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776252.220700:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e060700 x1862022710400640/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776252.220707:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e060700 x1862022710400640/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220713:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776252.220714:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776252.220716:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776252.220717:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776252.220720:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e060700 x1862022710400640/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220726:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776252.220730:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e060700 x1862022710400640/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220737:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776252.220739:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776252.220741:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776252.220742:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776252.220743:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776252.220745:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776252.220747:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776252.220751:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776252.220767:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776252.220771:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e060700 x1862022710400640/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776252.220786:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e060700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710400640:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776252.220790:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776252.220792:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776252.220794:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e063100 x1862022710400384/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220801:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776252.220804:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776252.220806:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776252.220809:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776252.220812:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776252.220814:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776252.220816:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776252.220818:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:1.0:1775776252.220821:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776252.220823:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776252.220825:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e063100. 00000100:00000001:1.0:1775776252.220829:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776252.220830:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776252.220832:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776252.220835:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e060e00 x1862022710400512/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220841:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776252.220842:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776252.220844:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776252.220846:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776252.220847:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776252.220849:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776252.220851:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776252.220852:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:1.0:1775776252.220854:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776252.220855:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776252.220856:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e060e00. 00000100:00000001:1.0:1775776252.220859:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776252.220860:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776252.220862:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776252.220864:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e060700 x1862022710400640/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776252.220870:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776252.220871:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776252.220872:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776252.220874:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776252.220875:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776252.220876:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776252.220878:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776252.220879:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:1.0:1775776252.220881:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776252.220882:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776252.220884:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e060700. 00000100:00000001:1.0:1775776252.220886:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776252.220887:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776252.220889:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776252.220896:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776252.220898:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776252.220900:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776252.220902:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776252.347195:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776252.347200:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776252.347203:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776252.347204:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776252.347206:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776252.347207:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776252.347252:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776252.347256:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776252.347258:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776253.243239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776253.243246:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776253.243248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776253.243253:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776253.243258:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776253.243261:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776253.243263:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776253.243266:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776253.243269:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776253.371250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776253.371254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776253.371258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776253.371259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776253.371261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776253.371262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776253.371305:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776253.371307:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776253.371310:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776254.267271:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776254.267278:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776254.267280:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776254.267285:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776254.267289:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776254.267294:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776254.267296:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776254.267299:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776254.267301:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776254.395284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776254.395291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776254.395298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776254.395299:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776254.395303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776254.395304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776254.395415:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776254.395420:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776254.395423:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776255.291215:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776255.291224:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776255.291226:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776255.291233:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776255.291237:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776255.291241:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776255.291249:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776255.291252:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776255.291256:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776255.419292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776255.419302:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776255.419304:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776255.419309:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776255.419314:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776255.419317:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776255.419320:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776255.419323:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776255.419328:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776255.419329:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776255.419332:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776255.419333:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776255.419388:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776255.419392:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776255.419395:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776256.123300:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307811078, at 4307811136 00000100:00000040:0.0:1775776256.123309:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307811078, at 4307811136 00000100:00000010:1.0:1775776256.123317:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2d20. 00000100:00000010:0.0:1775776256.123321:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2cc0. 00000100:00000001:1.0:1775776256.315228:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776256.315233:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776256.315235:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776256.315238:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776256.315241:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776256.315243:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776256.315246:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776256.315247:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776256.315249:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776256.443218:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776256.443222:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776256.443226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776256.443227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776256.443229:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776256.443230:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776256.443270:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776256.443273:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776256.443275:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776257.339234:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776257.339235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776257.339241:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1775776257.339242:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776257.339245:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776257.339245:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776257.339248:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776257.339252:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00100000:1.0:1775776257.339258:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776257.339261:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775776257.339264:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13139 00000100:00000001:1.0:1775776257.339265:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776257.339269:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776257.339272:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775776257.339273:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776257.339276:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776257.339280:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776257.339285:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776257.339287:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776257.339290:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776257.339291:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776257.339293:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776257.339296:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776257.339297:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776257.339299:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776257.339301:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776257.339302:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776257.339304:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776257.339306:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776257.339307:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776257.339308:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776257.339309:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776257.339311:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776257.339313:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776257.339315:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776257.339318:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776257.339320:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776257.339322:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776257.339323:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776257.339324:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776257.339331:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061880. 00000020:00000040:0.0:1775776257.339334:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776257.339337:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776257.339338:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776257.339340:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776257.339342:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776257.339344:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776257.339350:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000001:0.0:1775776257.339355:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776257.339361:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061880 x1862022710400768/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776257.339368:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776257.339372:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776257.339375:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005afd1f27] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061880 x1862022710400768/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776257.339396:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776257.339399:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776257.339402:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776257.339403:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776257.339405:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776257.339408:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776257.339410:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13139 00000400:00000200:0.0:1775776257.339414:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776257.339417:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776257.339420:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776257.339422:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1775776257.339423:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776257.339425:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776257.339427:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776257.339427:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776257.339431:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776257.339435:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776257.339437:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776257.339439:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000040:1.0:1775776257.339439:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061880 x1862022710400768/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775776257.339440:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776257.339444:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775776257.339448:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775776257.339453:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776257.339454:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776257.339457:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776257.339459:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776257.339461:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1775776257.339462:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776257.339464:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776257.339464:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776257.339469:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776257.339469:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775776257.339473:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00100000:1.0:1775776257.339475:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710400768:192.168.201.115@tcp:250:kworker.0 00000100:00080000:0.0:1775776257.339478:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776257.339483:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1775776257.339484:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776257.339486:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776257.339487:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776257.339489:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776257.339490:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776257.339491:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1775776257.339494:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776257.339496:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000010:1.0:1775776257.339497:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 00000008:00100000:0.0:1775776257.339500:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775776257.339500:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775776257.339504:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776257.339505:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776257.339508:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061c00. 00000400:00000200:1.0:1775776257.339508:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000040:0.0:1775776257.339511:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000400:00000010:1.0:1775776257.339511:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 02000000:00000001:0.0:1775776257.339512:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776257.339515:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776257.339516:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776257.339517:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1775776257.339518:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 02000000:00000001:0.0:1775776257.339519:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776257.339524:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6400. 00000100:00000200:1.0:1775776257.339525:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710400768, portal 25 00000100:00000001:0.0:1775776257.339527:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776257.339528:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776257.339530:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574559360 : -123823134992256 : ffff8f622e061880) 00000100:00100000:0.0:1775776257.339533:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061c00 x1862022710400896/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1775776257.339538:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061880 x1862022710400768/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776257.339551:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776257.339554:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776257.339555:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776257.339558:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000040:0.0:1775776257.339561:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076a44879] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061c00 x1862022710400896/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1775776257.339562:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776257.339567:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710400768, offset 0 00000100:00000001:0.0:1775776257.339573:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776257.339576:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00080000:0.0:1775776257.339577:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776257.339580:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776257.339582:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776257.339583:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775776257.339584:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776257.339588:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776257.339591:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13139 00000400:00000200:1.0:1775776257.339593:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776257.339598:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776257.339601:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776257.339604:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776257.339606:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776257.339609:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453683:453683:256:4294967295] 192.168.201.115@tcp LPNI seq info [453608:453608:8:4294967295] 00000400:00000200:1.0:1775776257.339612:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776257.339618:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000400:00000200:0.0:1775776257.339624:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000200:1.0:1775776257.339625:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776257.339630:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000800:00000010:1.0:1775776257.339630:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000100:00080000:0.0:1775776257.339632:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776257.339634:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776257.339636:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.0:1775776257.339639:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:0.0:1775776257.339640:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:1.0:1775776257.339643:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000100:00000001:0.0:1775776257.339644:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776257.339648:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776257.339650:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776257.339652:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:1.1:1775776257.339653:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000040:0.0:1775776257.339654:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000800:00000200:1.1:1775776257.339656:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000020:00000001:0.0:1775776257.339657:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775776257.339661:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775776257.339678:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776257.339679:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776257.339681:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776257.339683:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000100:1.0:1775776257.339683:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776257.339686:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000020:00000040:0.0:1775776257.339687:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775776257.339691:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776257.339692:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776257.339693:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776257.339693:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:0.0:1775776257.339697:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776257.339701:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776257.339701:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1775776257.339705:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776257.339707:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1775776257.339707:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000008:00000001:0.0:1775776257.339708:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776257.339711:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776257.339712:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000010:0.0:1775776257.339714:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061180. 00000020:00000040:0.0:1775776257.339716:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000100:00000001:1.0:1775776257.339718:0:2411:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1775776257.339719:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776257.339720:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776257.339723:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:1.0:1775776257.339723:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061880 x1862022710400768/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775776257.339726:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776257.339728:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776257.339734:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4400. 00000100:00000001:1.0:1775776257.339736:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1775776257.339737:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776257.339740:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061880 x1862022710400768/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776282 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776257.339744:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061180 x1862022710401024/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776257.339753:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776257.339754:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776257.339759:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e23859] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061180 x1862022710401024/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.339768:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776257.339769:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776257.339773:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776257.339773:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1775776257.339777:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776257.339779:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776257.339781:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776257.339781:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776257.339787:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13150) 00000400:00000200:1.0:1775776257.339791:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776257.339794:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307816198 00000100:00000010:1.0:1775776257.339798:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2d20. 00000100:00000001:1.0:1775776257.339804:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776257.339810:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307816198 00000100:00000010:1.0:1775776257.339811:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2ae0. 00000100:00000001:1.0:1775776257.339814:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776257.339816:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776257.339817:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776257.339819:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776257.339822:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776257/real 1775776257] req@ffff8f622e061880 x1862022710400768/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776257.339828:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776257.339829:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776257.339830:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776257.339832:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776257.339834:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776257.339835:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776257.339837:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061880 x1862022710400768/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776257.339842:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061880 x1862022710400768/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.339846:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776257.339847:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776257.339849:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776257.339851:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776257.339854:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061880 x1862022710400768/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.339860:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776257.339864:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061880 x1862022710400768/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.339871:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776257.339873:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776257.339874:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776257.339876:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776257.339877:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776257.339878:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776257.339880:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776257.339883:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776257.339884:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776257.339887:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061880 x1862022710400768/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776257.339892:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710400768:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776257.339894:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776257.339895:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776257.339897:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061880 x1862022710400768/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.339901:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776257.339902:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776257.339904:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776257.339905:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776257.339906:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776257.339907:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776257.339909:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776257.339911:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:1.0:1775776257.339914:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776257.339915:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776257.339916:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061880. 00000100:00000001:1.0:1775776257.339918:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776257.339919:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776257.339920:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776257.339924:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776257.339925:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776257.339926:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776257.339927:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776257.339928:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776257.339930:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061c00 x1862022710400896/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.339935:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776257.339936:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776257.339937:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776257.339938:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776257.339939:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776257.339941:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710400896:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776257.339942:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776257.339943:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776257.339944:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776257.339945:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776257.339947:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776257.339949:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776257.339951:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776257.339954:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776257.339956:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776257.339958:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710400896, portal 10 00000100:00000001:1.0:1775776257.339959:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776257.339960:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574560256 : -123823134991360 : ffff8f622e061c00) 00000100:00000040:1.0:1775776257.339963:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061c00 x1862022710400896/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.339967:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776257.339968:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776257.339970:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776257.339971:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710400896, offset 0 00000400:00000200:1.0:1775776257.339973:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776257.339975:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776257.339977:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776257.339979:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776257.339981:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776257.339982:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776257.339984:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776257.339986:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453684:453684:256:4294967295] 192.168.201.115@tcp LPNI seq info [453609:453609:8:4294967295] 00000400:00000200:1.0:1775776257.339988:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776257.339992:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776257.339995:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776257.339997:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776257.339999:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776257.340001:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1775776257.340004:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776257.340005:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000800:00000100:1.0:1775776257.340007:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776257.340008:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776257.340010:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776257.340012:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776257.340014:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776257.340015:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776257.340017:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776257.340018:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061c00 x1862022710400896/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.340023:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776257.340025:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061c00 x1862022710400896/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776282 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.340029:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776257.340029:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776257.340031:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776257.340033:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776257.340035:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776257.340036:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307812363 00000100:00000001:1.0:1775776257.340038:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307816198 00000100:00000001:1.0:1775776257.340039:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307812363 00000100:00000001:1.0:1775776257.340057:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307816198 00000100:00000001:1.0:1775776257.340059:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776257.340061:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776257.340062:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776257.340065:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776257/real 1775776257] req@ffff8f622e061c00 x1862022710400896/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776257.340073:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776257.340074:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776257.340076:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776257.340078:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776257.340079:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776257.340082:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776257.340084:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061c00 x1862022710400896/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776257.340093:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061c00 x1862022710400896/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.340100:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776257.340102:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776257.340103:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776257.340104:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776257.340108:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061c00 x1862022710400896/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.340114:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776257.340117:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061c00 x1862022710400896/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.340122:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776257.340123:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776257.340124:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776257.340125:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776257.340126:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776257.340130:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776257.340131:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776257.340149:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776257.340150:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776257.340153:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061c00 x1862022710400896/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776257.340158:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710400896:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776257.340161:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776257.340163:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061180 x1862022710401024/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.340167:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776257.340168:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776257.340169:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776257.340170:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776257.340172:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776257.340173:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710401024:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776257.340176:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776257.340177:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776257.340178:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776257.340179:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776257.340182:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776257.340184:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776257.340186:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776257.340188:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776257.340190:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776257.340192:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710401024, portal 10 00000100:00000001:1.0:1775776257.340193:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776257.340194:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574557568 : -123823134994048 : ffff8f622e061180) 00000100:00000040:1.0:1775776257.340199:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061180 x1862022710401024/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.340203:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776257.340205:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776257.340206:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776257.340207:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710401024, offset 0 00000400:00000200:1.0:1775776257.340209:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776257.340212:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776257.340214:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776257.340216:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776257.340217:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776257.340219:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776257.340220:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776257.340223:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453685:453685:256:4294967295] 192.168.201.115@tcp LPNI seq info [453610:453610:8:4294967295] 00000400:00000200:1.0:1775776257.340225:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776257.340229:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776257.340232:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776257.340233:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776257.340236:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776257.340238:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1775776257.340240:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776257.340241:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 52 secs later) 00000800:00000100:1.0:1775776257.340244:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776257.340245:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776257.340247:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776257.340250:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776257.340252:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776257.340253:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776257.340256:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776257.340267:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061180 x1862022710401024/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.340275:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776257.340278:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061180 x1862022710401024/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776282 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.340284:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776257.340286:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776257.340289:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776257.340292:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776257.340295:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776257.340297:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307812364 00000100:00000001:1.0:1775776257.340299:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307816198 00000100:00000001:1.0:1775776257.340302:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307812364 00000100:00000001:1.0:1775776257.340303:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307816198 00000100:00000001:1.0:1775776257.340305:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776257.340306:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776257.340307:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776257.340310:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776257/real 1775776257] req@ffff8f622e061180 x1862022710401024/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776257.340314:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776257.340315:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776257.340316:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776257.340318:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776257.340319:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776257.340320:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776257.340322:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061180 x1862022710401024/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776257.340326:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061180 x1862022710401024/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.340330:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776257.340331:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776257.340332:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776257.340332:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776257.340334:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061180 x1862022710401024/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.340337:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776257.340340:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061180 x1862022710401024/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.340344:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776257.340345:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776257.340346:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776257.340347:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776257.340348:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776257.340349:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776257.340350:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776257.340352:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776257.340353:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776257.340355:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061180 x1862022710401024/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776257.340359:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710401024:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776257.340361:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776257.340363:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776257.340365:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061c00 x1862022710400896/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.340371:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776257.340372:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776257.340374:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776257.340376:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776257.340383:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776257.340384:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776257.340386:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776257.340388:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:1.0:1775776257.340390:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776257.340391:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776257.340393:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061c00. 00000100:00000001:1.0:1775776257.340394:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776257.340395:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776257.340397:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776257.340399:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061180 x1862022710401024/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776282 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776257.340403:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776257.340404:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776257.340405:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776257.340406:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776257.340406:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776257.340407:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776257.340408:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776257.340409:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:1.0:1775776257.340410:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776257.340410:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776257.340411:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061180. 00000100:00000001:1.0:1775776257.340412:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776257.340413:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776257.340414:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776257.340416:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776257.340417:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776257.340418:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776257.340418:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776257.467241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776257.467245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776257.467250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776257.467252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776257.467255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776257.467256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776257.467317:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776257.467320:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776257.467323:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776258.363301:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776258.363308:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776258.363310:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776258.363315:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776258.363319:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776258.363322:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776258.363325:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776258.363328:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776258.363330:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776258.491197:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776258.491202:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776258.491205:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776258.491206:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776258.491208:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776258.491209:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776258.491255:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776258.491258:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776258.491261:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1775776259.195251:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:0.0:1775776259.195259:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:0.0:1775776259.195263:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1775776259.195438:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000200:1.0:1775776259.195443:0:2404:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1775776259.195452:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775776259.195456:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776259.195458:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776259.387233:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776259.387237:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776259.387239:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776259.387242:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776259.387245:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776259.387248:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776259.387251:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776259.387252:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776259.387254:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776259.515263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776259.515269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776259.515274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776259.515275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776259.515287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776259.515289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776259.515372:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776259.515376:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776259.515379:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776260.411286:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776260.411293:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776260.411294:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776260.411305:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776260.411309:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776260.411312:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776260.411314:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776260.411316:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776260.411318:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776260.539269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776260.539275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776260.539278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776260.539282:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776260.539283:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776260.539286:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776260.539288:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776260.539290:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776260.539291:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776260.539293:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776260.539297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776260.539304:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776260.539305:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776260.539306:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776260.539308:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776260.539309:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776260.539311:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776260.539312:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776260.539385:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776260.539389:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776260.539392:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776261.243224:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307816198, at 4307816256 00000100:00000040:0.0:1775776261.243233:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307816198, at 4307816256 00000100:00000010:1.0:1775776261.243239:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ae0. 00000100:00000010:0.0:1775776261.243247:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2d20. 00000100:00000001:1.0:1775776261.435247:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776261.435254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776261.435255:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776261.435260:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776261.435263:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776261.435265:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776261.435267:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776261.435269:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776261.435270:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776261.563276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776261.563280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776261.563284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776261.563284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776261.563287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776261.563290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776261.563356:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776261.563358:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776261.563361:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776262.459234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776262.459241:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776262.459242:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776262.459246:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775776262.459247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776262.459251:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776262.459253:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776262.459255:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776262.459256:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776262.459259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775776262.459259:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776262.459263:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776262.459264:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776262.459266:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776262.459270:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13145 00000400:00000200:3.0:1775776262.459278:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776262.459282:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776262.459284:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776262.459287:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776262.459291:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776262.459293:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776262.459296:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776262.459299:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776262.459301:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776262.459303:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776262.459306:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776262.459308:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776262.459318:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776262.459321:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776262.459323:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776262.459325:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776262.459327:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776262.459330:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776262.459333:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776262.459335:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776262.459338:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776262.459341:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776262.459344:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776262.459345:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776262.459348:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776262.459356:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775776262.459359:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776262.459362:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776262.459364:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776262.459366:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776262.459369:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776262.459372:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776262.459377:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000100:00000001:3.0:1775776262.459382:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776262.459388:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710401152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776262.459395:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776262.459398:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776262.459401:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710401152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776262.459415:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776262.459417:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776262.459420:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776262.459421:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776262.459422:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776262.459429:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776262.459431:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13145 00000400:00000200:3.0:1775776262.459435:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776262.459437:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776262.459439:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776262.459441:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776262.459443:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776262.459445:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776262.459446:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:1.0:1775776262.459448:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1775776262.459448:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776262.459451:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:1.0:1775776262.459452:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1775776262.459453:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775776262.459455:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1775776262.459456:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776262.459458:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776262.459460:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:1.0:1775776262.459463:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71180 x1862022710401152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776262.459464:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776262.459467:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776262.459470:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776262.459474:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776262.459477:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776262.459481:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776262.459482:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776262.459485:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776262.459485:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776262.459488:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776262.459489:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776262.459491:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776262.459492:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776262.459493:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776262.459497:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776262.459500:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710401152:192.168.201.115@tcp:250:kworker.0 00000100:00000001:3.0:1775776262.459500:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776262.459505:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00100000:3.0:1775776262.459505:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775776262.459508:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:3.0:1775776262.459508:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776262.459510:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776262.459511:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776262.459513:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776262.459518:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 00000100:00000010:3.0:1775776262.459518:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 02000000:00000001:1.0:1775776262.459521:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776262.459523:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776262.459527:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1775776262.459528:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776262.459531:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776262.459533:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000100:00000001:3.0:1775776262.459534:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776262.459537:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1775776262.459539:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 02000000:00000001:3.0:1775776262.459539:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776262.459544:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000100:00000200:1.0:1775776262.459546:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710401152, portal 25 00000100:00000001:3.0:1775776262.459548:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776262.459549:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776262.459551:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849008512 : -123823860543104 : ffff8f6202c71180) 00000100:00000040:1.0:1775776262.459558:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71180 x1862022710401152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776262.459558:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710401280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776262.459571:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776262.459584:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776262.459590:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776262.459593:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710401280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776262.459595:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776262.459599:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776262.459603:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710401152, offset 0 00000100:00000001:3.0:1775776262.459610:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776262.459614:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776262.459618:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00080000:3.0:1775776262.459618:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776262.459621:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776262.459623:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776262.459626:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776262.459630:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13145 00000400:00000200:1.0:1775776262.459634:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776262.459638:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776262.459644:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:3.0:1775776262.459644:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776262.459649:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1775776262.459653:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1775776262.459655:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776262.459659:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776262.459660:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000040:3.0:1775776262.459664:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776262.459666:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1775776262.459669:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776262.459672:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000020:00000001:3.0:1775776262.459674:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776262.459677:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776262.459680:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776262.459682:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776262.459684:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776262.459685:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453686:453686:256:4294967295] 192.168.201.115@tcp LPNI seq info [453611:453611:8:4294967295] 00000020:00000001:3.0:1775776262.459688:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776262.459695:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776262.459697:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776262.459698:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776262.459700:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776262.459702:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776262.459704:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776262.459709:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000400:00000200:1.0:1775776262.459713:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:3.0:1775776262.459714:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776262.459717:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776262.459721:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.0:1775776262.459725:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 02000000:00000001:3.0:1775776262.459725:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776262.459729:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776262.459731:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776262.459732:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775776262.459734:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000100:00000001:3.0:1775776262.459734:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776262.459740:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775776262.459743:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000800:00000200:1.0:1775776262.459745:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:3.0:1775776262.459747:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776262.459750:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776262.459751:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000100:00000001:3.0:1775776262.459752:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776262.459755:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776262.459758:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776262.459761:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000010:3.0:1775776262.459764:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000800:00000200:1.1:1775776262.459766:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000100:00000001:3.0:1775776262.459769:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1775776262.459774:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776262.459776:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000100:00100000:3.0:1775776262.459778:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710401408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776262.459782:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1775776262.459790:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775776262.459793:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1775776262.459794:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776262.459799:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776262.459803:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776262.459809:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1775776262.459810:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710401408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776262.459815:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71180 x1862022710401152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776262.459834:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776262.459836:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776262.459840:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71180 x1862022710401152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776287 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776262.459840:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776262.459849:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776262.459854:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776262.459855:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776262.459857:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1775776262.459861:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776262.459863:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776262.459866:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13155) 00000400:00000200:1.0:1775776262.459870:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776262.459873:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307821318 00000100:00000010:1.0:1775776262.459879:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2ae0. 00000100:00000001:1.0:1775776262.459885:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776262.459893:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307821318 00000100:00000010:1.0:1775776262.459896:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2c60. 00000100:00000001:1.0:1775776262.459899:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776262.459902:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776262.459904:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776262.459907:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776262.459911:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776262/real 1775776262] req@ffff8f6202c71180 x1862022710401152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776262.459919:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776262.459921:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776262.459923:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776262.459925:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776262.459927:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776262.459929:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776262.459933:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71180 x1862022710401152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776262.459942:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71180 x1862022710401152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776262.459949:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776262.459950:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776262.459952:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776262.459954:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776262.459957:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71180 x1862022710401152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776262.459963:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776262.459967:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71180 x1862022710401152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776262.459973:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776262.459976:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776262.459978:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776262.459979:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776262.459981:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776262.459983:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776262.459986:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776262.459990:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776262.459992:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776262.459996:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71180 x1862022710401152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776262.460002:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710401152:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776262.460006:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776262.460008:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776262.460010:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71180 x1862022710401152/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776262.460015:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776262.460018:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776262.460020:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776262.460022:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776262.460024:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776262.460025:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776262.460027:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776262.460029:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36c00. 02000000:00000001:1.0:1775776262.460031:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776262.460033:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776262.460035:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71180. 00000100:00000001:1.0:1775776262.460037:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776262.460038:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776262.460040:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776262.460046:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776262.460048:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776262.460049:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776262.460051:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776262.460052:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776262.460056:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72a00 x1862022710401280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776262.460062:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776262.460063:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776262.460065:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776262.460067:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776262.460069:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776262.460071:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710401280:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776262.460074:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776262.460076:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776262.460078:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776262.460079:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776262.460082:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776262.460084:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776262.460088:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776262.460090:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776262.460093:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776262.460096:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710401280, portal 10 00000100:00000001:1.0:1775776262.460098:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776262.460099:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849014784 : -123823860536832 : ffff8f6202c72a00) 00000100:00000040:1.0:1775776262.460105:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72a00 x1862022710401280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776262.460111:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776262.460113:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776262.460115:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776262.460117:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710401280, offset 0 00000400:00000200:1.0:1775776262.460121:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776262.460125:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776262.460129:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776262.460131:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776262.460154:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776262.460157:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776262.460159:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776262.460163:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453687:453687:256:4294967295] 192.168.201.115@tcp LPNI seq info [453612:453612:8:4294967295] 00000400:00000200:1.0:1775776262.460167:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776262.460173:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776262.460177:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776262.460180:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776262.460184:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776262.460186:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1775776262.460191:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776262.460193:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000100:1.0:1775776262.460197:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776262.460198:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776262.460201:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776262.460204:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776262.460207:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776262.460209:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776262.460212:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776262.460215:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72a00 x1862022710401280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776262.460223:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776262.460225:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72a00 x1862022710401280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776287 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776262.460232:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776262.460233:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776262.460236:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776262.460240:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776262.460243:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776262.460245:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307817484 00000100:00000001:1.0:1775776262.460248:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307821318 00000100:00000001:1.0:1775776262.460249:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307817484 00000100:00000001:1.0:1775776262.460251:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307821318 00000100:00000001:1.0:1775776262.460253:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776262.460254:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776262.460256:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776262.460258:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776262/real 1775776262] req@ffff8f6202c72a00 x1862022710401280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776262.460263:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776262.460264:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776262.460265:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776262.460267:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776262.460268:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776262.460269:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776262.460271:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72a00 x1862022710401280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776262.460276:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72a00 x1862022710401280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776262.460279:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776262.460280:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776262.460281:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776262.460282:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776262.460284:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72a00 x1862022710401280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776262.460287:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776262.460289:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72a00 x1862022710401280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776262.460293:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776262.460294:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776262.460295:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776262.460296:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776262.460298:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776262.460302:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776262.460304:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776262.460307:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776262.460309:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776262.460313:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72a00 x1862022710401280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776262.460320:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710401280:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776262.460323:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776262.460327:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72300 x1862022710401408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776262.460333:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776262.460334:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776262.460336:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776262.460338:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776262.460339:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776262.460342:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710401408:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776262.460345:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776262.460347:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776262.460348:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776262.460350:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776262.460353:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776262.460355:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776262.460357:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776262.460359:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776262.460361:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776262.460363:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710401408, portal 10 00000100:00000001:1.0:1775776262.460364:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776262.460365:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849012992 : -123823860538624 : ffff8f6202c72300) 00000100:00000040:1.0:1775776262.460369:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72300 x1862022710401408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776262.460373:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776262.460375:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776262.460377:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776262.460379:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710401408, offset 0 00000400:00000200:1.0:1775776262.460383:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776262.460387:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776262.460390:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776262.460393:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776262.460395:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776262.460397:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776262.460400:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776262.460402:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453688:453688:256:4294967295] 192.168.201.115@tcp LPNI seq info [453613:453613:8:4294967295] 00000400:00000200:1.0:1775776262.460406:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776262.460412:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776262.460415:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776262.460417:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776262.460420:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776262.460422:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1775776262.460424:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776262.460426:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 47 secs later) 00000800:00000100:1.0:1775776262.460428:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776262.460429:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776262.460431:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776262.460433:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776262.460435:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776262.460436:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776262.460437:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776262.460440:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72300 x1862022710401408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776262.460445:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776262.460446:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72300 x1862022710401408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776287 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776262.460450:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776262.460451:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776262.460453:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776262.460455:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776262.460457:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776262.460458:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307817484 00000100:00000001:1.0:1775776262.460460:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307821318 00000100:00000001:1.0:1775776262.460461:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307817484 00000100:00000001:1.0:1775776262.460462:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307821318 00000100:00000001:1.0:1775776262.460464:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776262.460465:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776262.460466:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776262.460468:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776262/real 1775776262] req@ffff8f6202c72300 x1862022710401408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776262.460472:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776262.460473:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776262.460474:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776262.460475:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776262.460476:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776262.460477:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776262.460479:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72300 x1862022710401408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776262.460483:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72300 x1862022710401408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776262.460487:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776262.460487:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776262.460488:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776262.460489:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776262.460491:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72300 x1862022710401408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776262.460494:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776262.460496:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72300 x1862022710401408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776262.460500:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776262.460502:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776262.460503:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776262.460503:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776262.460504:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776262.460505:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776262.460507:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776262.460508:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776262.460509:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776262.460511:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72300 x1862022710401408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776262.460515:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710401408:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776262.460517:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776262.460518:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776262.460520:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72a00 x1862022710401280/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776262.460523:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776262.460524:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776262.460525:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776262.460527:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776262.460527:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776262.460528:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776262.460529:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776262.460530:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35c00. 02000000:00000001:1.0:1775776262.460532:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776262.460533:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776262.460534:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72a00. 00000100:00000001:1.0:1775776262.460535:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776262.460536:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776262.460537:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776262.460539:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72300 x1862022710401408/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776287 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776262.460543:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776262.460543:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776262.460544:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776262.460545:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776262.460545:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776262.460546:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776262.460547:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776262.460548:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34000. 02000000:00000001:1.0:1775776262.460549:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776262.460549:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776262.460550:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72300. 00000100:00000001:1.0:1775776262.460551:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776262.460551:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776262.460552:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776262.460555:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776262.460555:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776262.460556:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776262.460557:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776262.587274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776262.587279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776262.587284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776262.587286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776262.587289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776262.587291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776262.587355:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776262.587358:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776262.587362:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776263.483247:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776263.483255:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776263.483256:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776263.483261:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776263.483265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776263.483269:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776263.483271:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776263.483274:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776263.483277:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776263.611256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776263.611261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776263.611266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776263.611268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776263.611272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776263.611273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776263.611341:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776263.611344:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776263.611347:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776264.507239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776264.507244:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776264.507246:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776264.507250:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776264.507253:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776264.507256:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776264.507259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776264.507261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776264.507264:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776264.635240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776264.635245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776264.635250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776264.635252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776264.635255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776264.635257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776264.635329:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776264.635333:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776264.635336:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776265.531269:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776265.531291:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776265.531293:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776265.531300:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776265.531309:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776265.531315:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776265.531318:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776265.531320:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776265.531322:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776265.659300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776265.659311:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776265.659313:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776265.659317:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776265.659323:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776265.659326:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776265.659328:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776265.659331:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776265.659335:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776265.659337:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776265.659341:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776265.659342:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776265.659401:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776265.659406:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776265.659409:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776266.363239:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307821318, at 4307821376 00000100:00000040:0.0:1775776266.363250:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307821318, at 4307821376 00000100:00000010:1.0:1775776266.363251:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2c60. 00000100:00000010:0.0:1775776266.363258:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ae0. 00000100:00000001:1.0:1775776266.555262:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776266.555269:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776266.555271:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776266.555276:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776266.555280:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776266.555283:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776266.555286:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776266.555288:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776266.555291:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776266.683237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776266.683241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776266.683245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776266.683247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776266.683250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776266.683251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776266.683345:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776266.683348:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776266.683350:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776267.579231:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776267.579236:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776267.579238:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776267.579240:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776267.579245:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1775776267.579246:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776267.579248:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776267.579251:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776267.579252:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776267.579254:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776267.579256:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776267.579261:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1775776267.579261:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13150 00000100:00000001:1.0:1775776267.579265:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775776267.579269:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776267.579272:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776267.579275:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776267.579278:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776267.579281:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776267.579284:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776267.579285:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776267.579287:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776267.579290:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776267.579291:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776267.579293:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776267.579295:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776267.579296:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776267.579298:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776267.579300:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776267.579301:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776267.579302:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776267.579303:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776267.579305:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776267.579306:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776267.579308:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776267.579310:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776267.579312:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776267.579314:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776267.579315:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776267.579317:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776267.579323:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775776267.579325:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776267.579328:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776267.579329:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776267.579330:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776267.579332:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776267.579334:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776267.579341:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000100:00000001:2.0:1775776267.579346:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776267.579353:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710401536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776267.579359:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776267.579364:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776267.579367:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710401536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776267.579381:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776267.579383:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776267.579385:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776267.579386:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776267.579387:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776267.579389:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776267.579390:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13150 00000400:00000200:2.0:1775776267.579392:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776267.579394:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776267.579396:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776267.579398:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776267.579400:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776267.579401:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776267.579403:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776267.579404:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776267.579404:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:1.0:1775776267.579406:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1775776267.579406:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776267.579408:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:1.0:1775776267.579410:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1775776267.579410:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776267.579411:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776267.579413:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776267.579414:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776267.579417:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776267.579418:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776267.579420:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:1.0:1775776267.579422:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344a680 x1862022710401536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1775776267.579422:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776267.579427:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776267.579433:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776267.579436:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776267.579440:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1775776267.579440:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776267.579442:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776267.579443:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776267.579446:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776267.579448:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:2.0:1775776267.579450:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775776267.579451:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1775776267.579453:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776267.579456:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776267.579456:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776267.579461:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000100:00100000:1.0:1775776267.579464:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710401536:192.168.201.115@tcp:250:kworker.0 00000020:00000040:2.0:1775776267.579464:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776267.579468:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776267.579469:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776267.579472:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1775776267.579472:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776267.579475:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776267.579476:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776267.579477:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000010:2.0:1775776267.579479:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 02000000:00000001:1.0:1775776267.579481:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776267.579482:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776267.579501:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710401664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776267.579506:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776267.579507:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776267.579509:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710401664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775776267.579512:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1775776267.579517:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776267.579519:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 00000100:00080000:2.0:1775776267.579519:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1775776267.579524:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776267.579524:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776267.579526:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776267.579529:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776267.579531:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776267.579533:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13150 00000400:00000200:1.0:1775776267.579535:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776267.579538:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776267.579541:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000200:2.0:1775776267.579541:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776267.579542:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776267.579547:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776267.579548:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776267.579550:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776267.579551:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776267.579552:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000010:1.0:1775776267.579553:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000020:00000040:2.0:1775776267.579553:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776267.579554:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776267.579556:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776267.579558:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776267.579559:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776267.579561:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000200:1.0:1775776267.579562:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710401536, portal 25 00000100:00000001:2.0:1775776267.579562:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776267.579565:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776267.579568:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776267.579568:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:1.0:1775776267.579571:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662545024 : -123823047006592 : ffff8f623344a680) 00000020:00000040:2.0:1775776267.579571:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776267.579574:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776267.579577:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776267.579578:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1775776267.579581:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344a680 x1862022710401536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775776267.579581:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776267.579584:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776267.579588:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776267.579590:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776267.579596:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776267.579597:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1775776267.579599:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776267.579603:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000010:2.0:1775776267.579605:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000400:00000010:1.0:1775776267.579607:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000020:00000040:2.0:1775776267.579609:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000100:00000200:1.0:1775776267.579612:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710401536, offset 0 02000000:00000001:2.0:1775776267.579612:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776267.579615:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776267.579617:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776267.579619:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1775776267.579621:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 02000000:00000001:2.0:1775776267.579623:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776267.579628:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000400:00000200:1.0:1775776267.579632:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:2.0:1775776267.579633:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776267.579639:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00100000:2.0:1775776267.579640:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710401792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776267.579646:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1775776267.579650:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775776267.579653:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:2.0:1775776267.579654:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776267.579659:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710401792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776267.579661:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776267.579667:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:2.0:1775776267.579672:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776267.579674:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453689:453689:256:4294967295] 192.168.201.115@tcp LPNI seq info [453614:453614:8:4294967295] 00000100:00000040:2.0:1775776267.579676:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776267.579682:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1775776267.579683:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:2.0:1775776267.579688:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776267.579692:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1775776267.579698:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000040:2.0:1775776267.579700:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13160) 00000800:00000200:1.0:1775776267.579707:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776267.579713:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776267.579720:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776267.579723:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775776267.579728:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776267.579732:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1775776267.579736:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776267.579739:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776267.579743:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776267.579748:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776267.579752:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776267.579755:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776267.579759:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776267.579763:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344a680 x1862022710401536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.579772:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776267.579775:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344a680 x1862022710401536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776292 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.579782:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776267.579784:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776267.579788:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776267.579792:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776267.579794:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307826438 00000100:00000010:1.0:1775776267.579798:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2c60. 00000100:00000001:1.0:1775776267.579803:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776267.579808:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307826438 00000100:00000010:1.0:1775776267.579809:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2540. 00000100:00000001:1.0:1775776267.579811:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776267.579813:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776267.579815:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776267.579816:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776267.579819:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776267/real 1775776267] req@ffff8f623344a680 x1862022710401536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776267.579827:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776267.579829:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776267.579832:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776267.579834:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776267.579835:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776267.579838:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776267.579841:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344a680 x1862022710401536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776267.579849:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344a680 x1862022710401536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.579855:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776267.579856:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776267.579858:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776267.579860:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776267.579864:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344a680 x1862022710401536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.579870:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776267.579874:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344a680 x1862022710401536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.579880:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776267.579883:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776267.579885:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776267.579886:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776267.579888:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776267.579890:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776267.579893:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776267.579897:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776267.579899:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776267.579903:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344a680 x1862022710401536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776267.579911:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710401536:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776267.579916:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776267.579917:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776267.579920:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344a680 x1862022710401536/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.579928:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776267.579931:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776267.579933:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776267.579936:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776267.579937:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776267.579938:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776267.579940:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776267.579942:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4d800. 02000000:00000001:1.0:1775776267.579946:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776267.579948:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776267.579950:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344a680. 00000100:00000001:1.0:1775776267.579953:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776267.579954:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776267.579956:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776267.579962:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776267.579963:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776267.579970:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776267.579971:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776267.579973:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776267.579977:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449f80 x1862022710401664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.579983:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776267.579985:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776267.579986:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776267.579988:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776267.579990:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776267.579992:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710401664:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776267.579995:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776267.579997:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776267.579999:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776267.580000:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776267.580003:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776267.580005:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776267.580009:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776267.580011:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776267.580014:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776267.580017:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710401664, portal 10 00000100:00000001:1.0:1775776267.580018:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776267.580020:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662543232 : -123823047008384 : ffff8f6233449f80) 00000100:00000040:1.0:1775776267.580025:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449f80 x1862022710401664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.580031:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776267.580033:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776267.580035:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776267.580037:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710401664, offset 0 00000400:00000200:1.0:1775776267.580041:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776267.580044:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776267.580047:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776267.580050:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776267.580052:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776267.580054:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776267.580057:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776267.580060:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453690:453690:256:4294967295] 192.168.201.115@tcp LPNI seq info [453615:453615:8:4294967295] 00000400:00000200:1.0:1775776267.580064:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776267.580070:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776267.580075:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776267.580078:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776267.580081:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776267.580084:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775776267.580088:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776267.580090:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1775776267.580094:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776267.580096:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776267.580098:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776267.580101:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776267.580104:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776267.580105:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776267.580108:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776267.580111:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449f80 x1862022710401664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.580117:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776267.580119:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449f80 x1862022710401664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776292 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.580123:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776267.580124:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776267.580126:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776267.580129:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776267.580131:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776267.580132:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307822603 00000100:00000001:1.0:1775776267.580149:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307826438 00000100:00000001:1.0:1775776267.580150:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307822604 00000100:00000001:1.0:1775776267.580151:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307826438 00000100:00000001:1.0:1775776267.580153:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776267.580154:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776267.580156:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776267.580158:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776267/real 1775776267] req@ffff8f6233449f80 x1862022710401664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776267.580163:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776267.580164:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776267.580165:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776267.580166:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776267.580167:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776267.580168:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776267.580170:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449f80 x1862022710401664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776267.580174:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449f80 x1862022710401664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.580178:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776267.580179:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776267.580180:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776267.580180:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776267.580182:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449f80 x1862022710401664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.580186:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776267.580187:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449f80 x1862022710401664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.580191:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776267.580192:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776267.580193:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776267.580194:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776267.580195:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776267.580197:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776267.580200:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776267.580202:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776267.580203:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776267.580205:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449f80 x1862022710401664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776267.580209:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710401664:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776267.580211:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776267.580213:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233448380 x1862022710401792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.580216:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776267.580217:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776267.580218:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776267.580219:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776267.580221:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776267.580222:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233448380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710401792:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776267.580224:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776267.580225:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776267.580225:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776267.580226:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776267.580228:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776267.580229:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776267.580231:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776267.580232:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776267.580234:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776267.580235:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710401792, portal 10 00000100:00000001:1.0:1775776267.580236:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776267.580237:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662536064 : -123823047015552 : ffff8f6233448380) 00000100:00000040:1.0:1775776267.580241:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233448380 x1862022710401792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.580248:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776267.580249:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776267.580251:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776267.580252:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710401792, offset 0 00000400:00000200:1.0:1775776267.580254:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776267.580256:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776267.580258:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776267.580259:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776267.580261:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776267.580263:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776267.580264:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776267.580266:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453691:453691:256:4294967295] 192.168.201.115@tcp LPNI seq info [453616:453616:8:4294967295] 00000400:00000200:1.0:1775776267.580268:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776267.580272:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776267.580274:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776267.580276:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776267.580278:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776267.580280:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775776267.580282:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776267.580284:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1775776267.580286:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776267.580287:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776267.580289:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776267.580291:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776267.580292:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776267.580293:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776267.580294:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776267.580296:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233448380 x1862022710401792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.580300:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776267.580302:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448380 x1862022710401792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776292 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.580310:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776267.580311:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776267.580313:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776267.580316:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776267.580319:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776267.580321:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307822604 00000100:00000001:1.0:1775776267.580323:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307826438 00000100:00000001:1.0:1775776267.580325:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307822604 00000100:00000001:1.0:1775776267.580326:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307826438 00000100:00000001:1.0:1775776267.580328:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776267.580330:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776267.580332:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776267.580335:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776267/real 1775776267] req@ffff8f6233448380 x1862022710401792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776267.580342:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776267.580343:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776267.580344:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776267.580346:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776267.580348:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776267.580349:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776267.580352:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233448380 x1862022710401792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776267.580359:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233448380 x1862022710401792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.580365:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776267.580366:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776267.580368:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776267.580369:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776267.580373:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233448380 x1862022710401792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.580380:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776267.580384:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233448380 x1862022710401792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.580390:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776267.580392:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776267.580394:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776267.580395:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776267.580396:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776267.580398:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776267.580401:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776267.580404:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776267.580406:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776267.580409:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233448380 x1862022710401792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776267.580416:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233448380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710401792:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776267.580420:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776267.580421:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776267.580424:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449f80 x1862022710401664/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.580430:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776267.580431:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776267.580433:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776267.580435:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776267.580436:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776267.580437:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776267.580438:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776267.580439:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4cc00. 02000000:00000001:1.0:1775776267.580441:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776267.580442:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776267.580443:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449f80. 00000100:00000001:1.0:1775776267.580445:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776267.580445:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776267.580446:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776267.580448:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233448380 x1862022710401792/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776292 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776267.580452:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776267.580453:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776267.580454:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776267.580455:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776267.580455:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776267.580456:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776267.580457:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776267.580458:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4d400. 02000000:00000001:1.0:1775776267.580459:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776267.580459:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776267.580460:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233448380. 00000100:00000001:1.0:1775776267.580461:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776267.580462:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776267.580462:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776267.580465:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776267.580465:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776267.580466:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776267.580467:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776267.707260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776267.707265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776267.707271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776267.707272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776267.707276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776267.707277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776267.707344:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776267.707348:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776267.707351:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776268.603238:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776268.603243:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776268.603245:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776268.603248:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776268.603251:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776268.603253:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776268.603256:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776268.603257:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776268.603259:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776268.731225:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776268.731230:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776268.731235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776268.731236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776268.731249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776268.731250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776268.731315:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776268.731319:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776268.731322:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776269.627222:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776269.627230:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776269.627231:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776269.627235:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776269.627238:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776269.627248:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776269.627252:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776269.627254:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776269.627255:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776269.755259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776269.755264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776269.755268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776269.755269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776269.755271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776269.755272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776269.755328:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776269.755332:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776269.755335:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776270.651221:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776270.651227:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776270.651229:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776270.651234:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776270.651238:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776270.651240:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776270.651242:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776270.651244:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776270.651246:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776270.779204:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776270.779215:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776270.779219:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776270.779222:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776270.779224:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776270.779226:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776270.779228:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776270.779230:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776270.779231:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776270.779233:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776270.779236:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776270.779236:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776270.779237:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776270.779238:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776270.779239:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776270.779240:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776270.779241:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776270.779242:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776270.779303:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776270.779305:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776270.779307:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776271.483234:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307826438, at 4307826496 00000100:00000040:0.0:1775776271.483245:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307826438, at 4307826496 00000100:00000010:1.0:1775776271.483248:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2540. 00000100:00000010:0.0:1775776271.483257:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2c60. 00000100:00000001:1.0:1775776271.675285:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776271.675291:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776271.675292:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776271.675296:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776271.675299:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776271.675302:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776271.675304:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776271.675306:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776271.675308:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776271.803237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776271.803243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776271.803248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776271.803249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776271.803253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776271.803254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776271.803307:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776271.803310:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776271.803313:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776272.699283:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776272.699291:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776272.699293:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776272.699299:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776272.699304:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776272.699307:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776272.699310:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776272.699313:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776272.699316:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776272.699446:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776272.699452:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776272.699455:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776272.699458:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776272.699462:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776272.699470:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13155 00000400:00000200:2.0:1775776272.699480:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776272.699487:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776272.699491:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776272.699495:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776272.699498:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776272.699500:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776272.699503:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776272.699507:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776272.699509:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776272.699512:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776272.699515:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776272.699517:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776272.699520:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776272.699523:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776272.699525:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776272.699527:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776272.699529:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776272.699532:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776272.699535:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776272.699538:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776272.699542:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776272.699545:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776272.699548:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776272.699551:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776272.699553:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776272.699563:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775776272.699567:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776272.699571:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776272.699573:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776272.699576:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776272.699579:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776272.699595:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776272.699609:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775776272.699616:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776272.699625:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710401920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776272.699637:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776272.699642:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776272.699647:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710401920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.699703:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776272.699706:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776272.699708:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776272.699714:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344aa00 x1862022710401920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.699723:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776272.699724:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776272.699728:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776272.699731:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776272.699734:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776272.699737:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710401920:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776272.699742:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776272.699746:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776272.699749:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776272.699751:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776272.699757:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776272.699759:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776272.699765:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776272.699770:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776272.699777:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776272.699784:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710401920, portal 25 00000100:00000001:1.0:1775776272.699786:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776272.699788:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662545920 : -123823047005696 : ffff8f623344aa00) 00000100:00000040:1.0:1775776272.699793:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344aa00 x1862022710401920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776297 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.699801:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776272.699805:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776272.699808:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776272.699811:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710401920, offset 0 00000400:00000200:1.0:1775776272.699819:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776272.699827:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776272.699843:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776272.699848:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776272.699852:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776272.699857:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776272.699860:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776272.699864:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453692:453692:256:4294967295] 192.168.201.115@tcp LPNI seq info [453617:453617:8:4294967295] 00000400:00000200:1.0:1775776272.699869:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776272.699879:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776272.699886:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776272.699891:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776272.699899:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776272.699903:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775776272.699909:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776272.699913:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000100:1.0:1775776272.699919:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776272.699921:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776272.699927:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776272.699932:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776272.699937:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776272.699940:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776272.699945:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776272.699950:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344aa00 x1862022710401920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776297 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776272.699980:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776272.699984:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776272.699988:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776272.699989:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776272.699991:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776272.699994:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776272.700008:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13155 00000400:00000200:2.0:1775776272.700013:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776272.700017:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776272.700020:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776272.700023:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776272.700025:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776272.700027:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776272.700030:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776272.700032:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776272.700034:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776272.700035:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776272.700038:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776272.700040:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776272.700042:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776272.700045:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776272.700047:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776272.700050:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776272.700051:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776272.700054:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776272.700057:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776272.700059:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776272.700061:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776272.700066:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776272.700068:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776272.700070:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776272.700074:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776272.700077:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776272.700079:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776272.700083:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775776272.700086:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776272.700088:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776272.700089:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776272.700091:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776272.700093:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776272.700095:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776272.700099:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775776272.700103:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776272.700108:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710402048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776272.700116:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776272.700119:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776272.700123:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710402048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.700175:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776272.700181:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344aa00 x1862022710401920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776297 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.700189:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776272.700191:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776272.700198:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776272.700203:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776272.700205:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307831558 00000100:00000010:1.0:1775776272.700210:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2540. 00000100:00000001:1.0:1775776272.700217:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776272.700223:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307831558 00000100:00000010:1.0:1775776272.700228:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2c60. 00000100:00000001:1.0:1775776272.700232:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776272.700235:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776272.700238:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776272.700240:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776272.700245:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776272/real 1775776272] req@ffff8f623344aa00 x1862022710401920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776272.700255:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776272.700256:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776272.700259:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776272.700262:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776272.700263:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776272.700266:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776272.700270:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344aa00 x1862022710401920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776272.700279:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344aa00 x1862022710401920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.700286:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776272.700288:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776272.700291:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776272.700293:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776272.700298:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344aa00 x1862022710401920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.700305:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776272.700310:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344aa00 x1862022710401920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.700319:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776272.700321:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776272.700323:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776272.700324:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776272.700327:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776272.700329:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776272.700331:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776272.700336:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776272.700338:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776272.700343:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344aa00 x1862022710401920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776272.700351:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710401920:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776272.700356:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776272.700358:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776272.700361:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344aa00 x1862022710401920/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.700368:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776272.700371:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776272.700373:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776272.700377:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776272.700378:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776272.700380:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776272.700382:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776272.700384:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4ec00. 02000000:00000001:1.0:1775776272.700388:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776272.700390:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776272.700392:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344aa00. 00000100:00000001:1.0:1775776272.700409:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776272.700411:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776272.700414:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776272.700420:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776272.700422:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776272.700425:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776272.700426:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776272.700428:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776272.700432:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449500 x1862022710402048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.700438:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776272.700440:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776272.700442:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776272.700444:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776272.700446:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776272.700449:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710402048:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776272.700453:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776272.700455:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776272.700456:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776272.700458:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776272.700461:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776272.700463:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776272.700467:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776272.700470:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776272.700473:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776272.700476:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710402048, portal 10 00000100:00000001:1.0:1775776272.700479:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776272.700480:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662540544 : -123823047011072 : ffff8f6233449500) 00000100:00000040:1.0:1775776272.700486:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449500 x1862022710402048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776297 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.700493:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776272.700496:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776272.700498:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776272.700500:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710402048, offset 0 00000400:00000200:1.0:1775776272.700504:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776272.700508:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776272.700513:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776272.700517:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776272.700520:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776272.700522:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776272.700525:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776272.700528:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453693:453693:256:4294967295] 192.168.201.115@tcp LPNI seq info [453618:453618:8:4294967295] 00000400:00000200:1.0:1775776272.700533:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776272.700540:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776272.700546:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776272.700549:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776272.700554:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776272.700557:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775776272.700562:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776272.700564:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000100:1.0:1775776272.700569:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776272.700571:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776272.700574:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776272.700578:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776272.700581:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776272.700584:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776272.700586:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776272.700590:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449500 x1862022710402048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776297 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.700599:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776272.700601:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449500 x1862022710402048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776297 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.700609:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776272.700611:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776272.700615:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776272.700619:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776272.700623:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776272.700625:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307827723 00000100:00000001:1.0:1775776272.700629:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307831558 00000100:00000001:1.0:1775776272.700631:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307827723 00000100:00000001:1.0:1775776272.700634:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307831558 00000100:00000001:1.0:1775776272.700636:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776272.700638:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776272.700640:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776272.700644:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776272/real 1775776272] req@ffff8f6233449500 x1862022710402048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776272.700652:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776272.700653:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776272.700655:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776272.700657:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776272.700659:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776272.700662:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776272.700665:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449500 x1862022710402048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776272.700674:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449500 x1862022710402048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.700680:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776272.700682:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776272.700684:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776272.700685:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776272.700689:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449500 x1862022710402048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.700696:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776272.700700:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449500 x1862022710402048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.700707:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776272.700710:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776272.700712:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776272.700714:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776272.700715:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776272.700720:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776272.700723:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776272.700726:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776272.700728:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776272.700732:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449500 x1862022710402048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776272.700740:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710402048:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776272.700744:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776272.700746:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776272.700749:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449500 x1862022710402048/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.700756:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776272.700757:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776272.700759:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776272.700762:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776272.700764:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776272.700765:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776272.700767:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776272.700769:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4e000. 02000000:00000001:1.0:1775776272.700772:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776272.700773:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776272.700775:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449500. 00000100:00000001:1.0:1775776272.700777:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776272.700779:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776272.700780:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776272.700784:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776272.700785:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776272.700787:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776272.700789:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776272.700857:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776272.700861:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776272.700865:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776272.700867:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776272.700869:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1775776272.700870:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776272.700872:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776272.700878:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776272.700883:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13155 00000400:00000200:2.0:1775776272.700888:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776272.700891:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776272.700894:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776272.700896:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776272.700899:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776272.700901:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776272.700903:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776272.700905:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776272.700907:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776272.700909:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776272.700911:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776272.700913:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776272.700915:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776272.700919:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776272.700921:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776272.700923:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776272.700925:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776272.700927:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776272.700930:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776272.700932:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776272.700934:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776272.700938:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776272.700940:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776272.700942:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776272.700945:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776272.700947:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776272.700950:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776272.700954:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775776272.700957:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776272.700959:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776272.700961:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776272.700963:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776272.700964:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776272.700966:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776272.700970:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775776272.700974:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776272.700979:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710402176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776272.700986:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776272.700989:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776272.700993:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710402176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776272.701011:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776272.701014:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776272.701018:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776272.701022:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776272.701026:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1775776272.701026:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776272.701033:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13165) 00000100:00000001:1.0:1775776272.701036:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776272.701038:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776272.701047:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344ad80 x1862022710402176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.701054:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776272.701055:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776272.701057:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776272.701059:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776272.701060:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776272.701063:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710402176:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776272.701066:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776272.701067:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776272.701069:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776272.701070:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776272.701073:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776272.701074:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776272.701077:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776272.701079:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776272.701082:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776272.701085:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710402176, portal 10 00000100:00000001:1.0:1775776272.701086:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776272.701087:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662546816 : -123823047004800 : ffff8f623344ad80) 00000100:00000040:1.0:1775776272.701092:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344ad80 x1862022710402176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776297 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.701096:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776272.701098:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776272.701100:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776272.701101:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710402176, offset 0 00000400:00000200:1.0:1775776272.701104:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776272.701107:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776272.701110:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776272.701113:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776272.701115:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776272.701117:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776272.701119:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776272.701121:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453694:453694:256:4294967295] 192.168.201.115@tcp LPNI seq info [453619:453619:8:4294967295] 00000400:00000200:1.0:1775776272.701124:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776272.701129:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776272.701147:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776272.701150:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776272.701153:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776272.701155:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775776272.701159:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776272.701161:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000100:1.0:1775776272.701163:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776272.701165:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776272.701168:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776272.701172:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776272.701176:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776272.701178:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776272.701181:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776272.701185:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344ad80 x1862022710402176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776297 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.701193:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776272.701197:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344ad80 x1862022710402176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776297 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.701204:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776272.701206:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776272.701210:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776272.701213:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776272.701217:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776272.701220:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307827725 00000100:00000001:1.0:1775776272.701222:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307831558 00000100:00000001:1.0:1775776272.701225:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307827725 00000100:00000001:1.0:1775776272.701227:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307831558 00000100:00000001:1.0:1775776272.701229:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776272.701231:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776272.701233:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776272.701236:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776272/real 1775776272] req@ffff8f623344ad80 x1862022710402176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776272.701242:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776272.701243:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776272.701244:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776272.701246:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776272.701247:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776272.701249:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776272.701251:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344ad80 x1862022710402176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776272.701257:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344ad80 x1862022710402176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.701261:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776272.701262:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776272.701263:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776272.701264:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776272.701266:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344ad80 x1862022710402176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.701270:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776272.701273:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344ad80 x1862022710402176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.701278:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776272.701280:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776272.701281:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776272.701282:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776272.701283:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776272.701285:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776272.701287:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776272.701289:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776272.701290:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776272.701293:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344ad80 x1862022710402176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776272.701298:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710402176:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776272.701301:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776272.701302:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776272.701304:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344ad80 x1862022710402176/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776297 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776272.701307:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776272.701309:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776272.701310:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776272.701311:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776272.701312:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776272.701313:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776272.701314:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776272.701316:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4f000. 02000000:00000001:1.0:1775776272.701317:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776272.701318:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776272.701319:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344ad80. 00000100:00000001:1.0:1775776272.701321:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776272.701322:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776272.701324:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776272.701326:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776272.701327:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776272.701329:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776272.701330:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776272.827294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776272.827303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776272.827309:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776272.827311:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776272.827315:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776272.827321:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776272.827449:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776272.827456:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776272.827462:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776273.723241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776273.723247:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776273.723249:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776273.723253:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776273.723257:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776273.723259:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776273.723261:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776273.723264:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776273.723267:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776273.851218:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776273.851222:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776273.851226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776273.851227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776273.851230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776273.851231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776273.851281:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776273.851284:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776273.851287:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776274.747217:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776274.747224:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776274.747226:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776274.747233:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776274.747236:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776274.747240:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776274.747243:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776274.747245:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776274.747247:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776274.875282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776274.875288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776274.875293:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776274.875295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776274.875299:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776274.875300:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776274.875377:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776274.875380:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776274.875383:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776275.771274:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776275.771283:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776275.771285:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776275.771291:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776275.771296:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776275.771302:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776275.771307:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776275.771310:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776275.771314:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776275.899203:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776275.899212:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776275.899214:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776275.899217:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776275.899222:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776275.899225:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776275.899227:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776275.899228:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776275.899232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776275.899233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776275.899236:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776275.899237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776275.899291:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776275.899294:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776275.899297:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776276.603271:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307831558, at 4307831616 00000100:00000010:1.0:1775776276.603287:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2c60. 00000100:00000040:0.0:1775776276.603297:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307831558, at 4307831616 00000100:00000010:0.0:1775776276.603315:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2540. 00000100:00000001:1.0:1775776276.795296:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776276.795307:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776276.795309:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776276.795318:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776276.795333:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776276.795338:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776276.795342:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776276.795345:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776276.795348:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776276.923273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776276.923280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776276.923285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776276.923285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776276.923288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776276.923289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776276.923366:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776276.923371:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776276.923374:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776277.819268:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775776277.819268:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776277.819275:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1775776277.819280:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776277.819284:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776277.819284:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1775776277.819290:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:3.0:1775776277.819290:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776277.819296:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776277.819297:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776277.819300:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776277.819303:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776277.819317:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776277.819320:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776277.819326:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13160 00000400:00000200:3.0:1775776277.819338:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776277.819346:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776277.819350:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776277.819353:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776277.819357:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776277.819359:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776277.819362:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776277.819366:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776277.819368:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776277.819371:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776277.819374:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776277.819377:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776277.819379:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776277.819382:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776277.819384:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776277.819386:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776277.819389:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776277.819392:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776277.819395:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776277.819398:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776277.819401:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776277.819405:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776277.819408:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776277.819410:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776277.819412:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776277.819423:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73b80. 00000020:00000040:3.0:1775776277.819426:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776277.819431:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776277.819433:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776277.819436:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776277.819439:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776277.819444:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776277.819453:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000100:00000001:3.0:1775776277.819461:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776277.819472:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73b80 x1862022710402304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776277.819483:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776277.819488:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776277.819493:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c52008f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73b80 x1862022710402304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776277.819514:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776277.819516:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776277.819520:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776277.819521:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776277.819523:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776277.819526:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776277.819530:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13160 00000400:00000200:3.0:1775776277.819534:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776277.819537:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1775776277.819538:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776277.819542:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776277.819543:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776277.819546:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776277.819547:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776277.819550:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776277.819553:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73b80 x1862022710402304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776277.819554:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776277.819560:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776277.819564:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775776277.819566:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1775776277.819567:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:1.0:1775776277.819568:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775776277.819570:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776277.819572:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 02000000:00000001:1.0:1775776277.819573:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776277.819576:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775776277.819576:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776277.819581:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775776277.819581:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00100000:1.0:1775776277.819587:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710402304:192.168.201.115@tcp:250:kworker.0 00000100:00000040:3.0:1775776277.819588:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776277.819592:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776277.819593:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1775776277.819596:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776277.819598:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1775776277.819599:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776277.819600:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776277.819603:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:1.0:1775776277.819604:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1775776277.819607:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776277.819610:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776277.819611:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:3.0:1775776277.819611:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776277.819614:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776277.819619:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775776277.819621:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776277.819623:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776277.819625:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000100:00000001:3.0:1775776277.819626:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776277.819632:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000010:1.0:1775776277.819633:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000008:00000001:3.0:1775776277.819636:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776277.819639:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776277.819641:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710402304, portal 25 00000100:00000001:1.0:1775776277.819644:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776277.819645:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849019264 : -123823860532352 : ffff8f6202c73b80) 00000100:00000010:3.0:1775776277.819645:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775776277.819649:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000040:1.0:1775776277.819653:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73b80 x1862022710402304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776302 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775776277.819654:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776277.819657:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776277.819661:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776277.819664:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776277.819667:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776277.819674:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000010:3.0:1775776277.819674:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000100:00000040:1.0:1775776277.819678:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776277.819679:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776277.819680:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776277.819684:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710402304, offset 0 00000100:00100000:3.0:1775776277.819686:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710402432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776277.819695:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776277.819700:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776277.819702:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776277.819706:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710402432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776277.819708:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776277.819718:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:3.0:1775776277.819719:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776277.819723:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776277.819726:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:3.0:1775776277.819728:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776277.819730:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776277.819732:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775776277.819733:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:3.0:1775776277.819737:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775776277.819740:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00080000:3.0:1775776277.819741:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13160 00000400:00000200:1.0:1775776277.819747:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:3.0:1775776277.819748:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776277.819753:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000400:00000200:1.0:1775776277.819757:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453695:453695:256:4294967295] 192.168.201.115@tcp LPNI seq info [453620:453620:8:4294967295] 00000100:00080000:3.0:1775776277.819758:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776277.819765:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776277.819768:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776277.819768:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776277.819773:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776277.819775:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776277.819777:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776277.819779:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776277.819782:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776277.819785:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000400:00000200:1.0:1775776277.819786:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000020:00000001:3.0:1775776277.819789:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776277.819793:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776277.819797:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000800:00000200:1.0:1775776277.819800:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776277.819800:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776277.819804:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776277.819806:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776277.819808:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000800:00000010:1.0:1775776277.819809:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000100:00080000:3.0:1775776277.819811:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776277.819815:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776277.819817:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776277.819819:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.0:1775776277.819822:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:3.0:1775776277.819823:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776277.819827:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:1.0:1775776277.819830:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000008:00100000:3.0:1775776277.819830:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776277.819835:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776277.819839:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776277.819839:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776277.819843:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000100:00000010:3.0:1775776277.819845:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000800:00000100:1.0:1775776277.819848:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776277.819851:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000020:00000040:3.0:1775776277.819851:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776277.819857:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776277.819858:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776277.819859:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1775776277.819860:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776277.819863:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776277.819865:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776277.819866:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 02000000:00000010:3.0:1775776277.819871:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000400:00000200:1.0:1775776277.819872:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776277.819876:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:3.0:1775776277.819876:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776277.819882:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710402560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776277.819883:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776277.819888:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73b80 x1862022710402304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776302 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776277.819897:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776277.819902:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776277.819909:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776277.819909:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710402560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776277.819916:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73b80 x1862022710402304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776302 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776277.819927:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776277.819933:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776277.819943:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776277.819945:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776277.819947:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776277.819948:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1775776277.819953:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776277.819956:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776277.819959:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13170) 00000400:00000200:1.0:1775776277.819963:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776277.819967:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307836678 00000100:00000010:1.0:1775776277.819973:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2c60. 00000100:00000001:1.0:1775776277.819981:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776277.819989:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307836678 00000100:00000010:1.0:1775776277.819991:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2ae0. 00000100:00000001:1.0:1775776277.819994:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776277.819996:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776277.819998:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776277.820000:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776277.820003:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776277/real 1775776277] req@ffff8f6202c73b80 x1862022710402304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776277.820011:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776277.820012:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776277.820014:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776277.820016:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776277.820017:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776277.820019:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776277.820022:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73b80 x1862022710402304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776277.820027:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73b80 x1862022710402304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776277.820033:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776277.820035:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776277.820037:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776277.820039:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776277.820042:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73b80 x1862022710402304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776277.820050:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776277.820055:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73b80 x1862022710402304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776277.820062:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776277.820066:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776277.820067:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776277.820068:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776277.820070:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776277.820071:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776277.820073:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776277.820076:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776277.820078:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776277.820081:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73b80 x1862022710402304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776277.820087:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710402304:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776277.820090:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776277.820091:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776277.820092:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73b80 x1862022710402304/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776277.820096:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776277.820098:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776277.820100:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776277.820102:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776277.820103:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776277.820104:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776277.820106:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776277.820108:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34000. 02000000:00000001:1.0:1775776277.820112:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776277.820113:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776277.820115:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73b80. 00000100:00000001:1.0:1775776277.820117:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776277.820118:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776277.820119:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776277.820126:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776277.820127:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776277.820128:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776277.820129:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776277.820130:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776277.820149:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72300 x1862022710402432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776277.820154:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776277.820155:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776277.820157:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776277.820159:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776277.820161:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776277.820163:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710402432:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776277.820167:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776277.820168:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776277.820170:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776277.820171:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776277.820174:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776277.820176:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776277.820180:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776277.820182:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776277.820186:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776277.820189:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710402432, portal 10 00000100:00000001:1.0:1775776277.820191:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776277.820192:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849012992 : -123823860538624 : ffff8f6202c72300) 00000100:00000040:1.0:1775776277.820199:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72300 x1862022710402432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776302 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776277.820206:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776277.820209:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776277.820212:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776277.820214:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710402432, offset 0 00000400:00000200:1.0:1775776277.820216:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776277.820219:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776277.820222:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776277.820224:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776277.820226:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776277.820227:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776277.820229:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776277.820231:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453696:453696:256:4294967295] 192.168.201.115@tcp LPNI seq info [453621:453621:8:4294967295] 00000400:00000200:1.0:1775776277.820233:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776277.820238:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776277.820241:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776277.820243:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776277.820245:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776277.820247:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775776277.820250:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776277.820252:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1775776277.820254:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776277.820255:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776277.820257:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776277.820259:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776277.820261:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776277.820262:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776277.820264:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776277.820266:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72300 x1862022710402432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776302 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776277.820271:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776277.820273:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72300 x1862022710402432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776302 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776277.820279:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776277.820281:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776277.820284:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776277.820288:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776277.820290:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776277.820292:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307832844 00000100:00000001:1.0:1775776277.820295:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307836678 00000100:00000001:1.0:1775776277.820297:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307832844 00000100:00000001:1.0:1775776277.820299:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307836678 00000100:00000001:1.0:1775776277.820301:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776277.820303:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776277.820305:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776277.820325:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776277/real 1775776277] req@ffff8f6202c72300 x1862022710402432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776277.820333:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776277.820335:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776277.820337:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776277.820339:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776277.820339:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776277.820341:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776277.820343:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72300 x1862022710402432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776277.820348:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72300 x1862022710402432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776277.820352:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776277.820353:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776277.820354:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776277.820354:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776277.820356:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72300 x1862022710402432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776277.820360:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776277.820362:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72300 x1862022710402432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776277.820367:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776277.820368:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776277.820369:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776277.820370:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776277.820371:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776277.820376:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776277.820379:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776277.820381:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776277.820383:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776277.820387:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72300 x1862022710402432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776277.820394:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710402432:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776277.820398:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776277.820401:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72a00 x1862022710402560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776277.820407:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776277.820408:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776277.820411:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776277.820412:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776277.820414:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776277.820418:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710402560:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776277.820421:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776277.820422:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776277.820424:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776277.820425:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776277.820429:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776277.820430:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776277.820434:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776277.820436:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776277.820440:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776277.820442:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710402560, portal 10 00000100:00000001:1.0:1775776277.820444:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776277.820446:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849014784 : -123823860536832 : ffff8f6202c72a00) 00000100:00000040:1.0:1775776277.820451:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72a00 x1862022710402560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776302 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776277.820458:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776277.820460:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776277.820462:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776277.820464:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710402560, offset 0 00000400:00000200:1.0:1775776277.820468:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776277.820472:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776277.820475:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776277.820477:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776277.820479:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776277.820481:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776277.820482:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776277.820484:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453697:453697:256:4294967295] 192.168.201.115@tcp LPNI seq info [453622:453622:8:4294967295] 00000400:00000200:1.0:1775776277.820487:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776277.820491:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776277.820494:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776277.820496:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776277.820498:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776277.820500:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775776277.820503:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776277.820505:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1775776277.820507:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776277.820509:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776277.820510:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776277.820513:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776277.820514:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776277.820516:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776277.820518:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776277.820520:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72a00 x1862022710402560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776302 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776277.820525:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776277.820527:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72a00 x1862022710402560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776302 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776277.820531:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776277.820532:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776277.820533:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776277.820535:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776277.820537:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776277.820538:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307832844 00000100:00000001:1.0:1775776277.820540:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307836678 00000100:00000001:1.0:1775776277.820541:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307832844 00000100:00000001:1.0:1775776277.820542:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307836678 00000100:00000001:1.0:1775776277.820544:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776277.820545:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776277.820546:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776277.820548:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776277/real 1775776277] req@ffff8f6202c72a00 x1862022710402560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776277.820552:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776277.820553:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776277.820554:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776277.820557:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776277.820558:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776277.820560:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776277.820563:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72a00 x1862022710402560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776277.820570:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72a00 x1862022710402560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776277.820578:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776277.820579:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776277.820580:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776277.820581:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776277.820583:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72a00 x1862022710402560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776277.820587:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776277.820590:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72a00 x1862022710402560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776277.820593:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776277.820595:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776277.820596:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776277.820597:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776277.820598:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776277.820600:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776277.820602:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776277.820605:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776277.820606:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776277.820610:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72a00 x1862022710402560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776277.820617:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710402560:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776277.820621:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776277.820623:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776277.820626:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72300 x1862022710402432/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776277.820633:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776277.820635:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776277.820636:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776277.820639:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776277.820641:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776277.820642:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776277.820644:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776277.820645:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35c00. 02000000:00000001:1.0:1775776277.820648:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776277.820649:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776277.820650:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72300. 00000100:00000001:1.0:1775776277.820652:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776277.820653:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776277.820654:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776277.820656:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72a00 x1862022710402560/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776302 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776277.820660:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776277.820661:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776277.820662:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776277.820664:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776277.820665:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776277.820666:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776277.820668:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776277.820669:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36c00. 02000000:00000001:1.0:1775776277.820671:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776277.820672:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776277.820674:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72a00. 00000100:00000001:1.0:1775776277.820676:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776277.820677:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776277.820679:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776277.820682:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776277.820683:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776277.820684:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776277.820686:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776277.947272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776277.947281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776277.947287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776277.947288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776277.947290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776277.947291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776277.947353:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776277.947358:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776277.947362:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776278.843270:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776278.843277:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776278.843278:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776278.843284:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776278.843302:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776278.843306:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776278.843309:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776278.843311:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776278.843313:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776278.971278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776278.971296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776278.971301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776278.971302:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776278.971305:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776278.971306:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776278.971371:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776278.971375:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776278.971379:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776279.867285:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776279.867294:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776279.867296:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776279.867302:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776279.867308:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776279.867312:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776279.867316:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776279.867319:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776279.867322:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776279.995263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776279.995297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776279.995307:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776279.995308:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776279.995311:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776279.995312:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776279.995368:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776279.995373:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776279.995377:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776280.891307:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776280.891313:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776280.891315:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776280.891321:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776280.891325:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776280.891330:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776280.891332:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776280.891334:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776280.891337:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776281.019280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776281.019287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776281.019294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776281.019297:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776281.019300:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776281.019304:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776281.019307:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776281.019309:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776281.019311:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776281.019314:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776281.019318:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776281.019320:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776281.019321:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776281.019323:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776281.019325:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776281.019326:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776281.019328:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776281.019330:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776281.019402:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776281.019407:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776281.019411:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776281.723252:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307836678, at 4307836736 00000100:00000040:0.0:1775776281.723264:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307836678, at 4307836736 00000100:00000010:1.0:1775776281.723265:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ae0. 00000100:00000010:0.0:1775776281.723273:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2c60. 00000100:00000001:3.0:1775776281.915315:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776281.915323:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776281.915325:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776281.915331:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776281.915336:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776281.915340:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776281.915343:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776281.915346:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776281.915349:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776282.043295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776282.043301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776282.043306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776282.043307:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776282.043312:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776282.043313:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776282.043374:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776282.043378:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776282.043381:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776282.939259:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776282.939268:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776282.939270:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776282.939274:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775776282.939276:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776282.939282:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775776282.939284:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776282.939287:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776282.939288:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776282.939293:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776282.939298:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775776282.939298:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776282.939303:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775776282.939307:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13165 00000100:00000001:3.0:1775776282.939308:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776282.939320:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776282.939329:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776282.939333:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776282.939337:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776282.939341:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776282.939344:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776282.939346:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776282.939350:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776282.939352:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776282.939361:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776282.939364:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776282.939366:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776282.939368:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776282.939370:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776282.939371:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776282.939373:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776282.939374:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776282.939376:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776282.939379:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776282.939380:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776282.939383:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776282.939385:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776282.939387:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776282.939389:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776282.939390:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776282.939397:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775776282.939400:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776282.939404:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776282.939405:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776282.939407:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776282.939409:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776282.939413:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776282.939419:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775776282.939425:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776282.939433:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710402688/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776282.939441:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776282.939444:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776282.939448:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710402688/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776282.939464:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776282.939467:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776282.939469:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776282.939471:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776282.939472:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776282.939474:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776282.939475:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13165 00000400:00000200:2.0:1775776282.939479:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776282.939481:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776282.939482:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776282.939484:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776282.939486:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776282.939488:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776282.939489:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776282.939491:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776282.939492:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776282.939493:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776282.939495:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776282.939496:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776282.939498:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776282.939500:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776282.939501:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776282.939503:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776282.939505:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776282.939507:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776282.939509:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776282.939510:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776282.939511:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776282.939514:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1775776282.939516:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776282.939520:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1775776282.939520:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776282.939521:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776282.939525:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000008:00100000:2.0:1775776282.939526:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776282.939529:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776282.939531:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776282.939534:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000100:00000040:1.0:1775776282.939535:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449180 x1862022710402688/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1775776282.939536:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776282.939540:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776282.939541:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776282.939545:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776282.939547:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776282.939550:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776282.939554:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:1.0:1775776282.939556:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776282.939557:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776282.939559:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776282.939564:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710402816/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1775776282.939565:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776282.939570:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776282.939572:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1775776282.939576:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776282.939576:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776282.939582:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710402816/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776282.939586:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710402688:192.168.201.115@tcp:250:kworker.0 00000100:00000001:2.0:1775776282.939595:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776282.939597:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776282.939598:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1775776282.939602:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776282.939603:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1775776282.939604:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1775776282.939605:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1775776282.939609:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776282.939609:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776282.939614:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13165 02000000:00000001:1.0:1775776282.939615:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:2.0:1775776282.939617:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776282.939620:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776282.939621:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776282.939623:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 02000000:00000010:1.0:1775776282.939624:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 00000100:00000001:2.0:1775776282.939625:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776282.939629:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776282.939630:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776282.939631:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776282.939635:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776282.939636:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776282.939637:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776282.939638:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776282.939642:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776282.939643:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000001:2.0:1775776282.939646:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776282.939649:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000010:1.0:1775776282.939650:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000100:00000001:2.0:1775776282.939651:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776282.939655:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776282.939656:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776282.939658:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776282.939659:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000400:00000010:1.0:1775776282.939660:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000001:2.0:1775776282.939661:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776282.939665:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776282.939668:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776282.939669:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776282.939670:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710402688, portal 25 00000100:00000001:2.0:1775776282.939671:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776282.939673:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775776282.939676:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000008:00000001:2.0:1775776282.939676:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776282.939680:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662539648 : -123823047011968 : ffff8f6233449180) 00000100:00000001:2.0:1775776282.939680:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776282.939688:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775776282.939691:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000100:00000040:1.0:1775776282.939693:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449180 x1862022710402688/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776307 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775776282.939694:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776282.939695:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776282.939698:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776282.939700:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776282.939702:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776282.939707:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775776282.939711:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776282.939713:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1775776282.939718:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710402944/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1775776282.939723:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:2.0:1775776282.939729:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1775776282.939730:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000001:2.0:1775776282.939731:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776282.939736:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710402688, offset 0 00000100:00000040:2.0:1775776282.939737:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710402944/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776282.939747:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:2.0:1775776282.939747:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776282.939751:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776282.939754:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776282.939758:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776282.939761:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1775776282.939764:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:2.0:1775776282.939765:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13175) 00000400:00000200:1.0:1775776282.939771:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776282.939776:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776282.939779:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776282.939783:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776282.939786:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776282.939789:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453698:453698:256:4294967295] 192.168.201.115@tcp LPNI seq info [453623:453623:8:4294967295] 00000400:00000200:1.0:1775776282.939792:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776282.939799:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776282.939803:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776282.939808:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776282.939814:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776282.939818:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775776282.939822:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776282.939824:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775776282.939827:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776282.939829:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776282.939834:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776282.939838:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776282.939842:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776282.939844:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776282.939848:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776282.939852:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449180 x1862022710402688/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776307 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.939858:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776282.939861:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449180 x1862022710402688/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776307 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.939866:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776282.939867:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776282.939872:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776282.939875:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776282.939877:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307841798 00000100:00000010:1.0:1775776282.939881:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2ae0. 00000100:00000001:1.0:1775776282.939886:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776282.939892:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307841798 00000100:00000010:1.0:1775776282.939895:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2d20. 00000100:00000001:1.0:1775776282.939898:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776282.939901:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776282.939904:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776282.939907:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776282.939911:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776282/real 1775776282] req@ffff8f6233449180 x1862022710402688/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776282.939921:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776282.939922:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776282.939924:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776282.939927:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776282.939929:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776282.939932:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776282.939936:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449180 x1862022710402688/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776282.939944:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449180 x1862022710402688/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.939951:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776282.939953:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776282.939956:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776282.939958:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776282.939962:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449180 x1862022710402688/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.939969:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776282.939974:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449180 x1862022710402688/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.939983:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776282.939985:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776282.939987:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776282.939989:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776282.939991:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776282.939993:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776282.939996:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776282.940001:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776282.940004:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776282.940008:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449180 x1862022710402688/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776282.940017:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710402688:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776282.940021:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776282.940024:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776282.940026:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449180 x1862022710402688/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.940033:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776282.940036:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776282.940039:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776282.940042:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776282.940044:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776282.940046:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776282.940048:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776282.940051:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4e800. 02000000:00000001:1.0:1775776282.940054:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776282.940056:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776282.940058:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449180. 00000100:00000001:1.0:1775776282.940061:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776282.940063:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776282.940065:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776282.940071:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776282.940073:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776282.940075:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776282.940077:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776282.940078:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776282.940082:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449c00 x1862022710402816/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.940089:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776282.940091:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776282.940093:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776282.940095:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776282.940098:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776282.940100:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710402816:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776282.940104:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776282.940106:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776282.940107:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776282.940109:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776282.940112:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776282.940114:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776282.940119:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776282.940121:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776282.940125:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776282.940127:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710402816, portal 10 00000100:00000001:1.0:1775776282.940129:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776282.940131:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662542336 : -123823047009280 : ffff8f6233449c00) 00000100:00000040:1.0:1775776282.940164:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449c00 x1862022710402816/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776307 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.940171:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776282.940174:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776282.940177:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776282.940179:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710402816, offset 0 00000400:00000200:1.0:1775776282.940183:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776282.940188:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776282.940192:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776282.940196:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776282.940198:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776282.940201:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776282.940204:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776282.940208:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453699:453699:256:4294967295] 192.168.201.115@tcp LPNI seq info [453624:453624:8:4294967295] 00000400:00000200:1.0:1775776282.940213:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776282.940237:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776282.940243:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776282.940246:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776282.940251:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776282.940254:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775776282.940258:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776282.940262:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775776282.940266:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776282.940268:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776282.940271:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776282.940275:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776282.940286:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776282.940288:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776282.940294:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776282.940299:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449c00 x1862022710402816/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776307 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.940308:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776282.940311:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449c00 x1862022710402816/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776307 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.940319:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776282.940321:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776282.940324:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776282.940328:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776282.940332:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776282.940335:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307837964 00000100:00000001:1.0:1775776282.940338:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307841798 00000100:00000001:1.0:1775776282.940341:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307837964 00000100:00000001:1.0:1775776282.940343:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307841798 00000100:00000001:1.0:1775776282.940345:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776282.940347:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776282.940349:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776282.940353:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776282/real 1775776282] req@ffff8f6233449c00 x1862022710402816/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776282.940362:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776282.940364:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776282.940366:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776282.940368:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776282.940370:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776282.940373:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776282.940376:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449c00 x1862022710402816/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776282.940387:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449c00 x1862022710402816/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.940395:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776282.940396:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776282.940398:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776282.940400:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776282.940404:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449c00 x1862022710402816/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.940411:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776282.940415:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449c00 x1862022710402816/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.940423:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776282.940426:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776282.940429:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776282.940430:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776282.940432:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776282.940437:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776282.940440:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776282.940444:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776282.940447:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776282.940451:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449c00 x1862022710402816/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776282.940460:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710402816:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776282.940464:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776282.940468:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233448700 x1862022710402944/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.940475:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776282.940477:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776282.940479:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776282.940481:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776282.940483:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776282.940487:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710402944:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776282.940490:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776282.940492:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776282.940494:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776282.940496:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776282.940499:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776282.940501:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776282.940505:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776282.940507:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776282.940511:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776282.940514:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710402944, portal 10 00000100:00000001:1.0:1775776282.940516:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776282.940518:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662536960 : -123823047014656 : ffff8f6233448700) 00000100:00000040:1.0:1775776282.940524:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233448700 x1862022710402944/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776307 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.940531:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776282.940533:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776282.940536:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776282.940538:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710402944, offset 0 00000400:00000200:1.0:1775776282.940542:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776282.940547:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776282.940550:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776282.940554:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776282.940556:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776282.940559:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776282.940562:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776282.940565:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453700:453700:256:4294967295] 192.168.201.115@tcp LPNI seq info [453625:453625:8:4294967295] 00000400:00000200:1.0:1775776282.940570:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776282.940577:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776282.940582:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776282.940586:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776282.940590:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776282.940593:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775776282.940598:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776282.940601:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775776282.940605:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776282.940607:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776282.940610:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776282.940614:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776282.940617:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776282.940619:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776282.940622:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776282.940626:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233448700 x1862022710402944/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776307 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.940640:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776282.940644:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448700 x1862022710402944/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776307 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.940651:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776282.940653:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776282.940656:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776282.940659:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776282.940662:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776282.940665:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307837964 00000100:00000001:1.0:1775776282.940667:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307841798 00000100:00000001:1.0:1775776282.940669:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307837964 00000100:00000001:1.0:1775776282.940672:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307841798 00000100:00000001:1.0:1775776282.940674:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776282.940676:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776282.940678:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776282.940682:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776282/real 1775776282] req@ffff8f6233448700 x1862022710402944/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776282.940691:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776282.940692:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776282.940694:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776282.940697:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776282.940699:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776282.940701:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776282.940704:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233448700 x1862022710402944/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776282.940714:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233448700 x1862022710402944/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.940722:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776282.940724:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776282.940726:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776282.940727:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776282.940731:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233448700 x1862022710402944/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.940739:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776282.940744:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233448700 x1862022710402944/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.940751:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776282.940754:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776282.940756:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776282.940757:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776282.940759:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776282.940761:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776282.940764:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776282.940767:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776282.940770:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776282.940774:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233448700 x1862022710402944/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776282.940783:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710402944:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776282.940787:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776282.940789:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776282.940792:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449c00 x1862022710402816/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.940800:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776282.940802:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776282.940804:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776282.940807:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776282.940808:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776282.940810:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776282.940812:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776282.940814:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4d400. 02000000:00000001:1.0:1775776282.940817:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776282.940818:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776282.940820:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449c00. 00000100:00000001:1.0:1775776282.940823:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776282.940824:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776282.940826:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776282.940830:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233448700 x1862022710402944/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776307 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776282.940837:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776282.940839:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776282.940840:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776282.940843:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776282.940844:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776282.940845:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776282.940847:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776282.940849:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4cc00. 02000000:00000001:1.0:1775776282.940851:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776282.940852:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776282.940854:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233448700. 00000100:00000001:1.0:1775776282.940856:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776282.940858:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776282.940860:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776282.940863:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776282.940865:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776282.940867:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776282.940868:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776283.067278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776283.067283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776283.067288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776283.067289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776283.067292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776283.067293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776283.067367:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776283.067372:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776283.067376:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776283.963267:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776283.963275:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776283.963276:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776283.963282:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776283.963286:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776283.963291:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776283.963294:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776283.963296:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776283.963299:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776284.091264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776284.091270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776284.091276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776284.091277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776284.091281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776284.091283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776284.091365:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776284.091369:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776284.091373:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776284.987248:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776284.987254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776284.987256:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776284.987261:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776284.987265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776284.987269:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776284.987272:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776284.987275:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776284.987277:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776285.115281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776285.115285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776285.115289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776285.115290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776285.115293:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776285.115294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776285.115349:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776285.115352:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776285.115355:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776286.011249:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776286.011256:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776286.011258:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776286.011263:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776286.011266:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776286.011269:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776286.011272:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776286.011273:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776286.011275:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776286.139245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776286.139251:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776286.139253:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776286.139256:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776286.139260:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776286.139262:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776286.139263:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776286.139265:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776286.139268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776286.139269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776286.139271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776286.139272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776286.139394:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776286.139401:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776286.139405:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776286.843246:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307841798, at 4307841856 00000100:00000040:0.0:1775776286.843256:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307841798, at 4307841856 00000100:00000010:1.0:1775776286.843258:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2d20. 00000100:00000010:0.0:1775776286.843264:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ae0. 00000100:00000001:3.0:1775776287.035210:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776287.035215:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776287.035216:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776287.035221:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776287.035224:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776287.035227:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776287.035229:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776287.035231:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776287.035233:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776287.163273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776287.163278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776287.163282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776287.163283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776287.163285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776287.163286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776287.163345:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776287.163349:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776287.163352:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776288.059239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776288.059247:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776288.059250:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776288.059256:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776288.059260:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776288.059264:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776288.059268:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775776288.059269:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775776288.059271:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775776288.059277:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776288.059277:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776288.059281:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776288.059284:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776288.059288:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776288.059303:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13170 00000400:00000200:0.0:1775776288.059311:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776288.059317:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776288.059319:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776288.059323:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776288.059325:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776288.059327:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776288.059328:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776288.059331:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776288.059332:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776288.059334:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776288.059336:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776288.059338:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776288.059339:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776288.059341:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776288.059342:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776288.059343:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776288.059345:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776288.059347:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776288.059350:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776288.059351:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776288.059353:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776288.059356:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776288.059358:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776288.059359:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776288.059364:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776288.059373:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061f80. 00000020:00000040:0.0:1775776288.059376:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776288.059379:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776288.059380:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776288.059382:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776288.059384:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776288.059387:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776288.059395:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4400. 00000100:00000001:0.0:1775776288.059400:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776288.059408:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061f80 x1862022710403072/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776288.059417:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776288.059420:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776288.059423:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000029d3794] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061f80 x1862022710403072/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776288.059440:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776288.059442:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776288.059444:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776288.059445:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776288.059447:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776288.059448:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776288.059450:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13170 00000400:00000200:0.0:1775776288.059452:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776288.059454:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776288.059456:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776288.059459:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776288.059460:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776288.059463:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776288.059464:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776288.059466:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776288.059467:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776288.059468:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776288.059470:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775776288.059471:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775776288.059472:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776288.059473:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1775776288.059474:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776288.059478:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776288.059478:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1775776288.059480:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776288.059482:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1775776288.059485:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776288.059489:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000040:1.0:1775776288.059491:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061f80 x1862022710403072/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775776288.059496:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776288.059503:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776288.059507:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1775776288.059508:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776288.059509:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776288.059513:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776288.059515:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776288.059515:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776288.059521:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776288.059522:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776288.059526:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710403072:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776288.059527:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776288.059532:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00100000:0.0:1775776288.059534:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775776288.059535:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776288.059538:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775776288.059541:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776288.059542:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1775776288.059546:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776288.059548:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 00000100:00000010:0.0:1775776288.059551:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063b80. 02000000:00000001:1.0:1775776288.059551:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775776288.059556:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000400:00000200:1.0:1775776288.059561:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:0.0:1775776288.059562:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776288.059565:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776288.059567:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000100:00000001:0.0:1775776288.059568:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776288.059572:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776288.059574:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776288.059574:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 02000000:00000010:0.0:1775776288.059583:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6400. 00000100:00000200:1.0:1775776288.059583:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710403072, portal 25 00000100:00000001:1.0:1775776288.059588:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776288.059590:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776288.059590:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574561152 : -123823134990464 : ffff8f622e061f80) 00000100:00000040:1.0:1775776288.059601:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061f80 x1862022710403072/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776288.059604:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063b80 x1862022710403200/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776288.059615:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776288.059621:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776288.059625:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000001:0.0:1775776288.059626:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:1.0:1775776288.059630:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710403072, offset 0 00000100:00000001:0.0:1775776288.059631:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776288.059638:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000040:0.0:1775776288.059640:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ee9007b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063b80 x1862022710403200/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776288.059648:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776288.059657:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776288.059658:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00080000:0.0:1775776288.059663:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776288.059665:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:0.0:1775776288.059671:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:1.0:1775776288.059671:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:0.0:1775776288.059675:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1775776288.059678:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00080000:0.0:1775776288.059679:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775776288.059682:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:0.0:1775776288.059686:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775776288.059688:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453701:453701:256:4294967295] 192.168.201.115@tcp LPNI seq info [453626:453626:8:4294967295] 00000100:00080000:0.0:1775776288.059693:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13170 00000400:00000200:1.0:1775776288.059695:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776288.059705:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776288.059713:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776288.059714:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000010:1.0:1775776288.059719:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000400:00000200:0.0:1775776288.059721:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776288.059724:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1775776288.059729:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:0.0:1775776288.059732:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000800:00000200:1.0:1775776288.059734:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000100:00000001:0.0:1775776288.059736:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776288.059740:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:1.1:1775776288.059744:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775776288.059745:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000200:1.1:1775776288.059748:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000020:00000001:0.0:1775776288.059749:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776288.059752:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776288.059756:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000100:1.0:1775776288.059757:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000020:00000040:0.0:1775776288.059760:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000800:00000010:1.0:1775776288.059761:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000020:00000001:0.0:1775776288.059766:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776288.059768:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:0.0:1775776288.059769:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775776288.059775:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776288.059776:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1775776288.059780:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776288.059783:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000100:00000040:0.0:1775776288.059784:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776288.059788:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000010:1.0:1775776288.059788:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000020:00000040:0.0:1775776288.059793:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00000001:1.0:1775776288.059795:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1775776288.059798:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000200:1.0:1775776288.059802:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061f80 x1862022710403072/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776288.059803:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776288.059808:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776288.059814:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776288.059815:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1775776288.059819:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776288.059820:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061f80 x1862022710403072/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776313 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776288.059824:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776288.059828:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775776288.059829:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775776288.059833:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776288.059834:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776288.059837:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776288.059840:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776288.059841:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061500. 00000020:00000040:0.0:1775776288.059845:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:1.0:1775776288.059848:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:0.0:1775776288.059849:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776288.059852:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776288.059852:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307846918 00000100:00000001:0.0:1775776288.059855:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776288.059857:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776288.059859:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776288.059859:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2d20. 02000000:00000010:0.0:1775776288.059863:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000001:1.0:1775776288.059867:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776288.059868:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776288.059874:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307846918 00000100:00100000:0.0:1775776288.059876:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061500 x1862022710403328/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776288.059885:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776288.059888:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776288.059888:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2cc0. 00000100:00000001:1.0:1775776288.059893:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000040:0.0:1775776288.059895:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb8d992] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061500 x1862022710403328/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.059896:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776288.059901:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776288.059906:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776288.059912:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776288/real 1775776288] req@ffff8f622e061f80 x1862022710403072/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776288.059914:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776288.059920:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:1.0:1775776288.059924:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:0.0:1775776288.059926:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000001:1.0:1775776288.059927:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776288.059930:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000100:00000040:0.0:1775776288.059931:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776288.059933:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776288.059936:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000040:0.0:1775776288.059939:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776288.059940:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776288.059945:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061f80 x1862022710403072/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776288.059947:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13180) 00000100:00000200:1.0:1775776288.059955:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061f80 x1862022710403072/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.059960:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776288.059962:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776288.059963:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776288.059965:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776288.059968:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061f80 x1862022710403072/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.059973:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776288.059976:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061f80 x1862022710403072/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.059981:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776288.059983:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776288.059984:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776288.059985:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776288.059992:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776288.059994:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776288.059996:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776288.059998:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776288.060000:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776288.060003:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061f80 x1862022710403072/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776288.060008:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710403072:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776288.060012:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776288.060013:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776288.060015:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061f80 x1862022710403072/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.060019:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776288.060020:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776288.060022:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776288.060024:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776288.060025:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776288.060026:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776288.060028:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776288.060029:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:1.0:1775776288.060032:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776288.060033:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776288.060034:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061f80. 00000100:00000001:1.0:1775776288.060036:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776288.060037:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776288.060038:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776288.060043:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776288.060044:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776288.060046:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776288.060047:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776288.060048:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776288.060050:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e063b80 x1862022710403200/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.060055:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776288.060056:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776288.060057:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776288.060059:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776288.060060:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776288.060062:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e063b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710403200:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776288.060064:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776288.060065:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776288.060066:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776288.060067:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776288.060069:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776288.060071:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776288.060073:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776288.060075:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776288.060077:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776288.060079:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710403200, portal 10 00000100:00000001:1.0:1775776288.060080:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776288.060081:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574568320 : -123823134983296 : ffff8f622e063b80) 00000100:00000040:1.0:1775776288.060086:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e063b80 x1862022710403200/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.060091:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776288.060092:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776288.060093:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776288.060095:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710403200, offset 0 00000400:00000200:1.0:1775776288.060098:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776288.060100:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776288.060102:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776288.060104:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776288.060106:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776288.060108:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776288.060110:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776288.060112:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453702:453702:256:4294967295] 192.168.201.115@tcp LPNI seq info [453627:453627:8:4294967295] 00000400:00000200:1.0:1775776288.060115:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776288.060119:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776288.060122:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776288.060124:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776288.060158:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776288.060161:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775776288.060164:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776288.060166:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775776288.060168:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776288.060169:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776288.060172:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776288.060174:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776288.060176:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776288.060178:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776288.060180:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776288.060182:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e063b80 x1862022710403200/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.060188:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776288.060190:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e063b80 x1862022710403200/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776313 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.060194:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776288.060195:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776288.060198:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776288.060200:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776288.060203:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776288.060205:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307843084 00000100:00000001:1.0:1775776288.060207:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307846918 00000100:00000001:1.0:1775776288.060208:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307843084 00000100:00000001:1.0:1775776288.060210:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307846918 00000100:00000001:1.0:1775776288.060212:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776288.060213:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776288.060215:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776288.060217:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776288/real 1775776288] req@ffff8f622e063b80 x1862022710403200/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776288.060222:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776288.060223:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776288.060224:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776288.060225:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776288.060226:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776288.060227:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776288.060229:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e063b80 x1862022710403200/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776288.060235:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e063b80 x1862022710403200/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.060239:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776288.060240:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776288.060241:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776288.060242:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776288.060244:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e063b80 x1862022710403200/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.060248:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776288.060250:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e063b80 x1862022710403200/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.060261:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776288.060262:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776288.060263:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776288.060264:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776288.060268:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776288.060271:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776288.060273:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776288.060276:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776288.060277:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776288.060281:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e063b80 x1862022710403200/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776288.060291:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e063b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710403200:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776288.060295:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776288.060298:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061500 x1862022710403328/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.060305:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776288.060307:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776288.060309:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776288.060311:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776288.060313:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776288.060316:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710403328:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776288.060319:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776288.060321:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776288.060323:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776288.060325:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776288.060328:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776288.060330:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776288.060333:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776288.060336:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776288.060339:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776288.060342:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710403328, portal 10 00000100:00000001:1.0:1775776288.060344:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776288.060345:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574558464 : -123823134993152 : ffff8f622e061500) 00000100:00000040:1.0:1775776288.060351:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061500 x1862022710403328/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.060364:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776288.060366:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776288.060368:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776288.060370:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710403328, offset 0 00000400:00000200:1.0:1775776288.060372:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776288.060375:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776288.060378:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776288.060380:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776288.060382:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776288.060383:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776288.060385:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776288.060387:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453703:453703:256:4294967295] 192.168.201.115@tcp LPNI seq info [453628:453628:8:4294967295] 00000400:00000200:1.0:1775776288.060390:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776288.060395:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776288.060398:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776288.060400:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776288.060403:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776288.060404:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775776288.060407:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776288.060409:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775776288.060411:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776288.060413:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776288.060415:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776288.060417:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776288.060419:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776288.060420:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776288.060422:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776288.060424:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061500 x1862022710403328/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.060430:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776288.060431:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061500 x1862022710403328/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776313 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.060440:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776288.060441:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776288.060444:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776288.060446:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776288.060448:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776288.060450:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307843084 00000100:00000001:1.0:1775776288.060451:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307846918 00000100:00000001:1.0:1775776288.060453:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307843084 00000100:00000001:1.0:1775776288.060454:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307846918 00000100:00000001:1.0:1775776288.060456:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776288.060457:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776288.060458:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776288.060460:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776288/real 1775776288] req@ffff8f622e061500 x1862022710403328/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776288.060465:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776288.060466:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776288.060467:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776288.060469:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776288.060470:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776288.060471:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776288.060473:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061500 x1862022710403328/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776288.060478:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061500 x1862022710403328/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.060482:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776288.060483:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776288.060484:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776288.060485:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776288.060487:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061500 x1862022710403328/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.060492:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776288.060495:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061500 x1862022710403328/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.060499:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776288.060500:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776288.060501:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776288.060502:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776288.060503:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776288.060504:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776288.060506:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776288.060508:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776288.060509:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776288.060511:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061500 x1862022710403328/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776288.060516:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710403328:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776288.060518:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776288.060519:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776288.060521:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e063b80 x1862022710403200/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.060525:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776288.060526:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776288.060527:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776288.060529:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776288.060529:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776288.060530:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776288.060532:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776288.060533:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:1.0:1775776288.060535:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776288.060536:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776288.060537:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e063b80. 00000100:00000001:1.0:1775776288.060539:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776288.060540:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776288.060541:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776288.060543:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061500 x1862022710403328/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776313 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776288.060547:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776288.060548:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776288.060549:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776288.060550:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776288.060551:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776288.060552:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776288.060553:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776288.060553:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:1.0:1775776288.060555:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776288.060555:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776288.060556:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061500. 00000100:00000001:1.0:1775776288.060557:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776288.060558:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776288.060559:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776288.060561:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776288.060562:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776288.060563:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776288.060564:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776288.187220:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776288.187224:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776288.187228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776288.187229:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776288.187232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776288.187233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776288.187312:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776288.187317:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776288.187321:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776289.083247:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776289.083252:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776289.083253:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776289.083259:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776289.083263:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776289.083267:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776289.083275:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776289.083279:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776289.083282:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776289.211259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776289.211266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776289.211273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776289.211274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776289.211291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776289.211292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776289.211367:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776289.211371:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776289.211375:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776290.107252:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776290.107266:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776290.107268:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776290.107274:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776290.107278:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776290.107282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776290.107285:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776290.107288:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776290.107291:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776290.235262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776290.235267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776290.235271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776290.235272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776290.235275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776290.235276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776290.235349:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776290.235354:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776290.235358:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776291.131301:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776291.131311:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776291.131313:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776291.131320:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776291.131325:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776291.131330:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776291.131333:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776291.131336:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776291.131339:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776291.259291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776291.259300:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776291.259308:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776291.259313:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776291.259316:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776291.259320:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776291.259323:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776291.259326:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776291.259328:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776291.259330:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776291.259335:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776291.259336:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776291.259338:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776291.259339:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776291.259341:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776291.259342:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776291.259344:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776291.259346:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776291.259406:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776291.259410:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776291.259414:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776291.963262:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307846918, at 4307846976 00000100:00000010:1.0:1775776291.963279:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2cc0. 00000100:00000040:0.0:1775776291.963281:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307846918, at 4307846976 00000100:00000010:0.0:1775776291.963292:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2d20. 00000100:00000001:3.0:1775776292.155218:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776292.155225:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776292.155226:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776292.155231:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776292.155234:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776292.155237:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776292.155240:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776292.155242:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776292.155244:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776292.283262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776292.283269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776292.283275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776292.283277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776292.283281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776292.283282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776292.283358:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776292.283363:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776292.283367:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776293.179252:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776293.179260:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776293.179262:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776293.179263:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775776293.179270:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:2.0:1775776293.179271:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776293.179276:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776293.179277:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775776293.179282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775776293.179283:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776293.179286:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776293.179289:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775776293.179290:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776293.179295:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776293.179300:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13175 00000400:00000200:2.0:1775776293.179309:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776293.179317:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776293.179321:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776293.179324:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776293.179328:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776293.179330:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776293.179333:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776293.179337:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776293.179339:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776293.179341:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776293.179344:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776293.179347:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776293.179349:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776293.179352:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776293.179354:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776293.179357:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776293.179359:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776293.179362:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776293.179365:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776293.179368:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776293.179371:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776293.179374:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776293.179377:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776293.179380:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776293.179382:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776293.179393:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775776293.179397:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776293.179401:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776293.179403:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776293.179407:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776293.179409:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776293.179413:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776293.179421:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775776293.179428:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776293.179437:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710403456/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776293.179448:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776293.179452:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776293.179457:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710403456/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776293.179484:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776293.179487:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776293.179491:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776293.179492:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776293.179493:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776293.179497:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776293.179499:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13175 00000400:00000200:2.0:1775776293.179503:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776293.179508:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776293.179510:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776293.179514:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776293.179517:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776293.179522:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776293.179525:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1775776293.179527:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776293.179528:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1775776293.179532:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:1.0:1775776293.179536:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344bb80 x1862022710403456/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775776293.179537:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776293.179539:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776293.179543:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775776293.179548:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:2.0:1775776293.179548:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:1.0:1775776293.179551:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776293.179553:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776293.179555:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776293.179559:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1775776293.179559:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776293.179564:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776293.179567:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00100000:1.0:1775776293.179570:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710403456:192.168.201.115@tcp:250:kworker.0 00000100:00000001:2.0:1775776293.179573:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776293.179579:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776293.179580:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1775776293.179584:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776293.179585:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776293.179589:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775776293.179589:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:1.0:1775776293.179592:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1775776293.179595:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000010:1.0:1775776293.179598:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 00000100:00000001:2.0:1775776293.179599:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776293.179602:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776293.179603:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776293.179609:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:2.0:1775776293.179611:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1775776293.179612:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 02000000:00000001:2.0:1775776293.179618:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776293.179620:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776293.179624:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776293.179627:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776293.179628:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000001:2.0:1775776293.179632:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776293.179636:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710403456, portal 25 00000100:00000010:2.0:1775776293.179639:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000100:00000001:1.0:1775776293.179640:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776293.179642:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662550400 : -123823047001216 : ffff8f623344bb80) 00000020:00000040:2.0:1775776293.179644:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000040:1.0:1775776293.179649:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344bb80 x1862022710403456/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775776293.179649:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776293.179652:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776293.179657:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776293.179660:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776293.179663:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776293.179665:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000010:2.0:1775776293.179669:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000040:1.0:1775776293.179670:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:2.0:1775776293.179675:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776293.179676:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776293.179682:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710403456, offset 0 00000100:00100000:2.0:1775776293.179685:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710403584/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776293.179695:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:2.0:1775776293.179700:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776293.179703:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776293.179708:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710403584/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776293.179716:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:2.0:1775776293.179724:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776293.179725:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00080000:2.0:1775776293.179728:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776293.179731:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:2.0:1775776293.179735:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:1.0:1775776293.179737:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:2.0:1775776293.179739:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776293.179741:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775776293.179742:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776293.179747:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:2.0:1775776293.179747:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776293.179754:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13175 00000400:00000200:1.0:1775776293.179755:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453704:453704:256:4294967295] 192.168.201.115@tcp LPNI seq info [453629:453629:8:4294967295] 00000400:00000200:2.0:1775776293.179764:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776293.179765:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776293.179770:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776293.179773:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776293.179779:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776293.179785:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776293.179789:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776293.179791:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776293.179793:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:1.0:1775776293.179794:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000020:00000040:2.0:1775776293.179794:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776293.179798:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776293.179801:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776293.179804:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:1.0:1775776293.179809:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000020:00000001:2.0:1775776293.179810:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776293.179816:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000800:00000010:1.0:1775776293.179818:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000100:00000001:2.0:1775776293.179819:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776293.179823:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776293.179826:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000200:1.0:1775776293.179830:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000040:2.0:1775776293.179831:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000800:00000200:1.0:1775776293.179835:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00080000:2.0:1775776293.179838:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776293.179844:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776293.179844:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:2.0:1775776293.179848:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:1.1:1775776293.179849:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775776293.179853:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 02000000:00000001:2.0:1775776293.179853:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776293.179858:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775776293.179859:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000100:00000001:2.0:1775776293.179864:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775776293.179866:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00100000:2.0:1775776293.179866:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776293.179871:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776293.179875:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776293.179878:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000010:2.0:1775776293.179878:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775776293.179883:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:1.0:1775776293.179885:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 02000000:00000001:2.0:1775776293.179886:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000010:1.0:1775776293.179888:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 02000000:00000001:2.0:1775776293.179889:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776293.179893:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776293.179895:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775776293.179896:0:2411:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1775776293.179897:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776293.179903:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344bb80 x1862022710403456/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1775776293.179912:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:1.0:1775776293.179918:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1775776293.179918:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776293.179925:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344bb80 x1862022710403456/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776318 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776293.179928:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710403712/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776293.179951:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776293.179953:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776293.179955:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776293.179957:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710403712/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.179958:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776293.179965:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776293.179971:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776293.179974:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:2.0:1775776293.179976:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776293.179977:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307852038 00000100:00000010:1.0:1775776293.179984:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2cc0. 00000100:00000040:2.0:1775776293.179984:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776293.179990:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776293.179992:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000040:2.0:1775776293.179995:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776293.180000:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307852038 00000100:00000040:2.0:1775776293.180002:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13185) 00000100:00000010:1.0:1775776293.180004:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e21e0. 00000100:00000001:1.0:1775776293.180008:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776293.180016:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776293.180018:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776293.180021:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776293.180025:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776293/real 1775776293] req@ffff8f623344bb80 x1862022710403456/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776293.180033:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776293.180034:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776293.180036:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776293.180054:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776293.180056:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776293.180058:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776293.180061:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344bb80 x1862022710403456/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776293.180068:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344bb80 x1862022710403456/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.180072:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776293.180073:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776293.180075:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776293.180077:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776293.180079:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344bb80 x1862022710403456/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.180084:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776293.180088:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344bb80 x1862022710403456/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.180092:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776293.180094:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776293.180095:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776293.180096:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776293.180098:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776293.180099:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776293.180102:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776293.180104:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776293.180106:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776293.180108:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344bb80 x1862022710403456/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776293.180113:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710403456:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776293.180116:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776293.180118:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776293.180119:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344bb80 x1862022710403456/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.180124:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776293.180126:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776293.180128:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776293.180129:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776293.180131:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776293.180132:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776293.180155:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776293.180157:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4c000. 02000000:00000001:1.0:1775776293.180159:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776293.180160:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776293.180162:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344bb80. 00000100:00000001:1.0:1775776293.180164:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776293.180165:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776293.180167:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776293.180172:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776293.180173:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776293.180175:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776293.180176:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776293.180177:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776293.180179:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233448700 x1862022710403584/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.180185:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776293.180186:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776293.180188:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776293.180189:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776293.180191:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776293.180192:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710403584:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776293.180194:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776293.180196:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776293.180197:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776293.180198:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776293.180201:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776293.180202:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776293.180205:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776293.180207:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776293.180210:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776293.180212:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710403584, portal 10 00000100:00000001:1.0:1775776293.180213:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776293.180214:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662536960 : -123823047014656 : ffff8f6233448700) 00000100:00000040:1.0:1775776293.180219:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233448700 x1862022710403584/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.180223:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776293.180225:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776293.180226:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776293.180228:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710403584, offset 0 00000400:00000200:1.0:1775776293.180231:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776293.180233:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776293.180236:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776293.180238:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776293.180239:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776293.180241:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776293.180243:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776293.180246:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453705:453705:256:4294967295] 192.168.201.115@tcp LPNI seq info [453630:453630:8:4294967295] 00000400:00000200:1.0:1775776293.180249:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776293.180253:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776293.180256:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776293.180258:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776293.180261:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776293.180263:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775776293.180266:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776293.180268:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775776293.180271:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776293.180272:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776293.180274:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776293.180277:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776293.180279:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776293.180280:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776293.180281:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776293.180283:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233448700 x1862022710403584/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.180289:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776293.180291:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448700 x1862022710403584/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776318 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.180295:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776293.180296:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776293.180298:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776293.180301:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776293.180303:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776293.180304:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307848204 00000100:00000001:1.0:1775776293.180307:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307852038 00000100:00000001:1.0:1775776293.180308:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307848204 00000100:00000001:1.0:1775776293.180310:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307852038 00000100:00000001:1.0:1775776293.180311:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776293.180314:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776293.180315:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776293.180317:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776293/real 1775776293] req@ffff8f6233448700 x1862022710403584/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776293.180322:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776293.180323:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776293.180324:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776293.180325:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776293.180326:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776293.180328:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776293.180330:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233448700 x1862022710403584/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776293.180335:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233448700 x1862022710403584/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.180339:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776293.180340:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776293.180341:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776293.180342:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776293.180344:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233448700 x1862022710403584/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.180347:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776293.180350:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233448700 x1862022710403584/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.180354:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776293.180355:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776293.180356:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776293.180357:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776293.180358:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776293.180361:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776293.180363:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776293.180365:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776293.180367:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776293.180369:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233448700 x1862022710403584/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776293.180374:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710403584:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776293.180376:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776293.180378:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449c00 x1862022710403712/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.180382:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776293.180383:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776293.180384:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776293.180385:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776293.180386:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776293.180388:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710403712:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776293.180391:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776293.180392:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776293.180393:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776293.180394:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776293.180396:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776293.180397:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776293.180399:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776293.180400:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776293.180402:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776293.180404:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710403712, portal 10 00000100:00000001:1.0:1775776293.180405:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776293.180406:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662542336 : -123823047009280 : ffff8f6233449c00) 00000100:00000040:1.0:1775776293.180409:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449c00 x1862022710403712/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.180413:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776293.180415:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776293.180416:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776293.180418:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710403712, offset 0 00000400:00000200:1.0:1775776293.180420:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776293.180423:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776293.180425:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776293.180427:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776293.180429:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776293.180430:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776293.180432:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776293.180435:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453706:453706:256:4294967295] 192.168.201.115@tcp LPNI seq info [453631:453631:8:4294967295] 00000400:00000200:1.0:1775776293.180440:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776293.180446:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776293.180450:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776293.180453:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776293.180457:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776293.180460:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775776293.180464:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776293.180467:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775776293.180471:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776293.180473:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776293.180476:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776293.180480:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776293.180483:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776293.180485:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776293.180488:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776293.180492:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449c00 x1862022710403712/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.180499:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776293.180502:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449c00 x1862022710403712/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776318 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.180506:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776293.180508:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776293.180510:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776293.180512:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776293.180515:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776293.180516:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307848204 00000100:00000001:1.0:1775776293.180518:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307852038 00000100:00000001:1.0:1775776293.180519:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307848204 00000100:00000001:1.0:1775776293.180520:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307852038 00000100:00000001:1.0:1775776293.180522:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776293.180523:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776293.180524:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776293.180526:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776293/real 1775776293] req@ffff8f6233449c00 x1862022710403712/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776293.180532:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776293.180533:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776293.180534:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776293.180535:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776293.180536:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776293.180538:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776293.180540:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449c00 x1862022710403712/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776293.180545:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449c00 x1862022710403712/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.180549:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776293.180550:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776293.180551:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776293.180552:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776293.180554:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449c00 x1862022710403712/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.180558:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776293.180561:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449c00 x1862022710403712/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.180566:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776293.180567:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776293.180569:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776293.180569:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776293.180570:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776293.180571:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776293.180573:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776293.180575:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776293.180576:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776293.180578:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449c00 x1862022710403712/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776293.180583:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710403712:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776293.180585:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776293.180586:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776293.180588:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233448700 x1862022710403584/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.180593:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776293.180594:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776293.180596:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776293.180597:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776293.180598:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776293.180599:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776293.180600:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776293.180601:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4f400. 02000000:00000001:1.0:1775776293.180603:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776293.180604:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776293.180605:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233448700. 00000100:00000001:1.0:1775776293.180606:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776293.180608:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776293.180609:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776293.180610:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449c00 x1862022710403712/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776318 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776293.180615:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776293.180615:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776293.180616:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776293.180617:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776293.180618:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776293.180619:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776293.180620:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776293.180620:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4ec00. 02000000:00000001:1.0:1775776293.180622:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776293.180622:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776293.180623:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449c00. 00000100:00000001:1.0:1775776293.180624:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776293.180625:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776293.180626:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776293.180628:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776293.180629:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776293.180630:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776293.180631:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776293.307211:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776293.307217:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776293.307224:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776293.307225:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776293.307228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776293.307230:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776293.307289:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776293.307293:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776293.307297:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776294.203188:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776294.203196:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776294.203198:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776294.203203:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776294.203207:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776294.203210:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776294.203213:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776294.203216:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776294.203218:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776294.331246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776294.331254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776294.331260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776294.331261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776294.331264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776294.331266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776294.331366:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776294.331373:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776294.331377:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776295.227253:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776295.227264:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776295.227266:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776295.227272:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776295.227277:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776295.227281:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776295.227284:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776295.227286:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776295.227289:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776295.355228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776295.355235:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776295.355237:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776295.355246:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776295.355250:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776295.355252:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776295.355253:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776295.355255:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776295.355257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776295.355258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776295.355261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776295.355262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776295.355351:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776295.355356:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776295.355361:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776296.251279:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776296.251285:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776296.251287:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776296.251291:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776296.251294:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776296.251297:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776296.251299:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776296.251301:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776296.251303:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776296.379282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776296.379289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776296.379295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776296.379296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776296.379300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776296.379301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776296.379375:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776296.379379:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776296.379383:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776297.083265:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307852038, at 4307852096 00000800:00000200:0.0:1775776297.083277:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000100:00000010:1.0:1775776297.083280:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e21e0. 00000800:00000200:0.0:1775776297.083284:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:0.0:1775776297.083288:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000100:00000040:0.0:1775776297.083318:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307852038, at 4307852096 00000100:00000010:0.0:1775776297.083325:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2cc0. 00000800:00000200:0.0:1775776297.083467:0:2404:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000001:1.0:1775776297.083468:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1775776297.083485:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1775776297.083490:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1775776297.083494:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776297.275268:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776297.275274:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776297.275276:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776297.275281:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776297.275284:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776297.275288:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776297.275291:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776297.275293:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776297.275295:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776297.403212:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776297.403221:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776297.403227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776297.403228:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776297.403231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776297.403231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776297.403295:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776297.403299:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776297.403303:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775776298.299247:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775776298.299247:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776298.299253:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776298.299255:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776298.299260:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:3.0:1775776298.299260:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776298.299265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776298.299266:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775776298.299268:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775776298.299271:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776298.299271:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776298.299274:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776298.299278:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775776298.299278:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776298.299287:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13180 00000400:00000200:2.0:1775776298.299297:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776298.299304:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776298.299308:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776298.299311:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776298.299314:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776298.299316:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776298.299319:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776298.299322:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776298.299324:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776298.299327:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776298.299330:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776298.299332:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776298.299335:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776298.299338:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776298.299340:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776298.299342:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776298.299344:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776298.299347:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776298.299350:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776298.299353:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776298.299357:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776298.299360:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776298.299363:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776298.299365:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776298.299367:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776298.299375:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775776298.299378:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776298.299383:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776298.299384:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776298.299387:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776298.299390:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776298.299394:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776298.299402:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775776298.299409:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776298.299417:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710403840/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776298.299427:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776298.299431:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776298.299435:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710403840/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776298.299458:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776298.299461:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776298.299465:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776298.299466:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776298.299468:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776298.299470:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776298.299473:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13180 00000400:00000200:2.0:1775776298.299477:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776298.299480:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776298.299484:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776298.299487:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776298.299490:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776298.299492:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776298.299495:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776298.299495:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776298.299498:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775776298.299500:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000040:2.0:1775776298.299501:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:1.0:1775776298.299503:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1775776298.299505:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776298.299509:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000040:1.0:1775776298.299511:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449180 x1862022710403840/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775776298.299512:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776298.299518:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776298.299525:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776298.299528:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776298.299528:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776298.299533:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1775776298.299533:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775776298.299537:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1775776298.299537:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776298.299542:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776298.299543:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775776298.299548:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00100000:1.0:1775776298.299549:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710403840:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776298.299555:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1775776298.299555:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776298.299559:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776298.299560:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775776298.299563:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776298.299564:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776298.299567:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1775776298.299570:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:1.0:1775776298.299572:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:2.0:1775776298.299574:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776298.299575:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776298.299579:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775776298.299583:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000008:00100000:2.0:1775776298.299584:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000010:1.0:1775776298.299587:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000008:00000001:2.0:1775776298.299589:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776298.299592:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776298.299593:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000010:2.0:1775776298.299597:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000100:00000200:1.0:1775776298.299600:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710403840, portal 25 00000020:00000040:2.0:1775776298.299601:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000001:1.0:1775776298.299604:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1775776298.299606:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1775776298.299607:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662539648 : -123823047011968 : ffff8f6233449180) 02000000:00000001:2.0:1775776298.299610:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776298.299614:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449180 x1862022710403840/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776298.299614:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776298.299618:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776298.299622:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776298.299624:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776298.299627:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 02000000:00000010:2.0:1775776298.299627:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000400:00000010:1.0:1775776298.299632:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000001:2.0:1775776298.299633:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776298.299635:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710403840, offset 0 00000400:00000200:1.0:1775776298.299644:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00100000:2.0:1775776298.299644:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710403968/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776298.299654:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:2.0:1775776298.299659:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775776298.299662:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:2.0:1775776298.299663:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776298.299671:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1775776298.299672:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710403968/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776298.299676:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776298.299681:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776298.299686:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776298.299693:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453707:453707:256:4294967295] 192.168.201.115@tcp LPNI seq info [453632:453632:8:4294967295] 00000100:00000001:2.0:1775776298.299694:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776298.299698:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00080000:2.0:1775776298.299700:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776298.299704:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776298.299709:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1775776298.299713:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00080000:2.0:1775776298.299713:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776298.299718:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776298.299723:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13180 00000800:00000200:1.0:1775776298.299727:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000400:00000200:2.0:1775776298.299732:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776298.299737:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000800:00000010:1.0:1775776298.299739:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000100:00080000:2.0:1775776298.299740:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776298.299745:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776298.299748:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.0:1775776298.299752:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775776298.299754:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:1.0:1775776298.299758:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 11 secs later) 00000100:00000001:2.0:1775776298.299759:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776298.299765:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776298.299766:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776298.299770:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:1.1:1775776298.299771:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000040:2.0:1775776298.299773:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000800:00000200:1.1:1775776298.299777:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 11 secs later) 00000020:00000001:2.0:1775776298.299778:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776298.299783:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000100:1.0:1775776298.299788:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000040:2.0:1775776298.299790:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000800:00000010:1.0:1775776298.299792:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000100:00000001:2.0:1775776298.299794:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776298.299797:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776298.299799:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1775776298.299801:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776298.299806:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776298.299810:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00080000:2.0:1775776298.299814:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000400:00000200:1.0:1775776298.299818:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000100:00000001:2.0:1775776298.299819:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776298.299822:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 02000000:00000001:2.0:1775776298.299823:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776298.299828:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776298.299831:0:2411:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1775776298.299831:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776298.299835:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000200:1.0:1775776298.299839:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449180 x1862022710403840/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1775776298.299839:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776298.299846:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776298.299851:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776298.299857:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000100:00000001:1.0:1775776298.299860:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000040:2.0:1775776298.299862:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000100:00000040:1.0:1775776298.299867:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449180 x1862022710403840/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776323 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775776298.299868:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776298.299870:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776298.299875:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776298.299878:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776298.299882:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776298.299883:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776298.299887:0:2411:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000010:2.0:1775776298.299888:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:1.0:1775776298.299892:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776298.299893:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776298.299899:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00100000:2.0:1775776298.299902:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710404096/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776298.299904:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307857158 00000100:00000010:1.0:1775776298.299912:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e21e0. 00000100:00000001:2.0:1775776298.299914:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776298.299917:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776298.299920:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000040:2.0:1775776298.299923:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710404096/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.299929:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307857158 00000100:00000010:1.0:1775776298.299936:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e29c0. 00000100:00000001:2.0:1775776298.299937:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776298.299943:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000040:2.0:1775776298.299943:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776298.299964:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776298.299966:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1775776298.299968:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776298.299972:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776298.299975:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776298.299977:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000040:2.0:1775776298.299982:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13191) 00000100:00100000:1.0:1775776298.299984:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776298/real 1775776298] req@ffff8f6233449180 x1862022710403840/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776298.299995:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776298.299998:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776298.300001:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776298.300004:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776298.300006:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776298.300009:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776298.300012:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449180 x1862022710403840/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776298.300018:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449180 x1862022710403840/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.300023:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776298.300024:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776298.300025:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776298.300027:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776298.300029:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449180 x1862022710403840/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.300034:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776298.300037:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449180 x1862022710403840/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.300042:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776298.300043:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776298.300044:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776298.300046:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776298.300048:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776298.300049:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776298.300051:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776298.300054:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776298.300056:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776298.300058:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449180 x1862022710403840/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776298.300064:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710403840:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776298.300067:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776298.300069:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776298.300071:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449180 x1862022710403840/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.300075:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776298.300077:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776298.300078:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776298.300080:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776298.300081:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776298.300082:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776298.300084:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776298.300085:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4ec00. 02000000:00000001:1.0:1775776298.300088:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776298.300089:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776298.300090:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449180. 00000100:00000001:1.0:1775776298.300093:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776298.300094:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776298.300095:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776298.300101:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776298.300102:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776298.300104:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776298.300105:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776298.300106:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776298.300108:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344ad80 x1862022710403968/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.300113:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776298.300114:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776298.300116:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776298.300117:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776298.300119:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776298.300120:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710403968:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776298.300122:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776298.300124:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776298.300125:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776298.300126:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776298.300128:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776298.300129:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776298.300154:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776298.300157:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776298.300160:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776298.300162:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710403968, portal 10 00000100:00000001:1.0:1775776298.300163:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776298.300164:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662546816 : -123823047004800 : ffff8f623344ad80) 00000100:00000040:1.0:1775776298.300169:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344ad80 x1862022710403968/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.300173:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776298.300175:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776298.300176:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776298.300178:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710403968, offset 0 00000400:00000200:1.0:1775776298.300180:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776298.300183:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776298.300186:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776298.300188:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776298.300190:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776298.300191:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776298.300193:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776298.300195:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453708:453708:256:4294967295] 192.168.201.115@tcp LPNI seq info [453633:453633:8:4294967295] 00000400:00000200:1.0:1775776298.300198:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776298.300203:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776298.300206:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776298.300208:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776298.300211:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776298.300213:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1775776298.300216:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776298.300218:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 11 secs later) 00000800:00000100:1.0:1775776298.300221:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776298.300222:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776298.300224:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776298.300226:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776298.300228:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776298.300229:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776298.300231:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776298.300233:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344ad80 x1862022710403968/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.300238:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776298.300240:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344ad80 x1862022710403968/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776323 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.300244:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776298.300245:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776298.300247:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776298.300250:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776298.300252:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776298.300254:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307853324 00000100:00000001:1.0:1775776298.300256:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307857158 00000100:00000001:1.0:1775776298.300257:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307853324 00000100:00000001:1.0:1775776298.300259:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307857158 00000100:00000001:1.0:1775776298.300260:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776298.300262:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776298.300263:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776298.300265:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776298/real 1775776298] req@ffff8f623344ad80 x1862022710403968/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776298.300271:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776298.300272:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776298.300273:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776298.300274:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776298.300275:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776298.300276:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776298.300278:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344ad80 x1862022710403968/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776298.300283:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344ad80 x1862022710403968/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.300287:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776298.300288:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776298.300289:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776298.300290:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776298.300292:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344ad80 x1862022710403968/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.300296:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776298.300298:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344ad80 x1862022710403968/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.300302:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776298.300303:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776298.300304:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776298.300305:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776298.300306:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776298.300309:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776298.300311:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776298.300313:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776298.300315:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776298.300317:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344ad80 x1862022710403968/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776298.300322:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710403968:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776298.300324:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776298.300326:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449500 x1862022710404096/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.300330:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776298.300331:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776298.300332:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776298.300333:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776298.300334:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776298.300336:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710404096:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776298.300338:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776298.300339:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776298.300341:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776298.300342:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776298.300344:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776298.300345:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776298.300347:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776298.300348:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776298.300350:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776298.300352:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710404096, portal 10 00000100:00000001:1.0:1775776298.300353:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776298.300354:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662540544 : -123823047011072 : ffff8f6233449500) 00000100:00000040:1.0:1775776298.300357:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449500 x1862022710404096/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.300361:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776298.300363:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776298.300364:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776298.300365:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710404096, offset 0 00000400:00000200:1.0:1775776298.300368:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776298.300370:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776298.300373:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776298.300376:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776298.300379:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776298.300381:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776298.300384:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776298.300387:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453709:453709:256:4294967295] 192.168.201.115@tcp LPNI seq info [453634:453634:8:4294967295] 00000400:00000200:1.0:1775776298.300391:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776298.300397:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776298.300402:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776298.300405:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776298.300409:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776298.300412:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1775776298.300416:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776298.300419:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 11 secs later) 00000800:00000100:1.0:1775776298.300423:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776298.300425:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776298.300428:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776298.300431:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776298.300434:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776298.300436:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776298.300438:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776298.300442:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449500 x1862022710404096/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.300449:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776298.300452:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449500 x1862022710404096/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776323 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.300458:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776298.300460:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776298.300463:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776298.300466:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776298.300469:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776298.300471:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307853324 00000100:00000001:1.0:1775776298.300473:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307857158 00000100:00000001:1.0:1775776298.300475:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307853324 00000100:00000001:1.0:1775776298.300477:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307857158 00000100:00000001:1.0:1775776298.300479:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776298.300481:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776298.300483:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776298.300486:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776298/real 1775776298] req@ffff8f6233449500 x1862022710404096/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776298.300493:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776298.300495:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776298.300496:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776298.300498:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776298.300500:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776298.300502:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776298.300504:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449500 x1862022710404096/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776298.300512:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449500 x1862022710404096/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.300518:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776298.300519:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776298.300520:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776298.300521:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776298.300525:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449500 x1862022710404096/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.300532:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776298.300536:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449500 x1862022710404096/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.300542:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776298.300544:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776298.300546:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776298.300547:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776298.300549:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776298.300551:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776298.300553:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776298.300557:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776298.300558:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776298.300563:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449500 x1862022710404096/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776298.300570:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710404096:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776298.300574:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776298.300575:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776298.300578:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344ad80 x1862022710403968/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.300585:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776298.300586:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776298.300588:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776298.300591:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776298.300592:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776298.300593:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776298.300596:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776298.300598:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4f400. 02000000:00000001:1.0:1775776298.300600:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776298.300601:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776298.300603:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344ad80. 00000100:00000001:1.0:1775776298.300607:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776298.300608:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776298.300610:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776298.300613:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449500 x1862022710404096/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776323 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776298.300619:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776298.300620:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776298.300622:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776298.300624:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776298.300625:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776298.300626:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776298.300628:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776298.300629:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4c000. 02000000:00000001:1.0:1775776298.300631:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776298.300632:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776298.300633:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449500. 00000100:00000001:1.0:1775776298.300634:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776298.300635:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776298.300637:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776298.300640:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776298.300642:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776298.300643:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776298.300645:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776298.427279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776298.427285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776298.427289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776298.427290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776298.427294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776298.427294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776298.427361:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776298.427365:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776298.427369:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776299.323222:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776299.323229:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776299.323231:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776299.323237:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776299.323242:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776299.323246:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776299.323249:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776299.323252:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776299.323255:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776299.451284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776299.451291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776299.451298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776299.451299:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776299.451304:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776299.451305:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776299.451373:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776299.451376:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776299.451378:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776300.347211:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776300.347217:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776300.347218:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776300.347224:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776300.347227:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776300.347231:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776300.347233:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776300.347235:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776300.347238:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776300.475276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776300.475282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776300.475287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776300.475290:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776300.475292:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776300.475296:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776300.475299:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776300.475301:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776300.475303:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776300.475306:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776300.475309:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776300.475311:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776300.475312:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776300.475314:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776300.475315:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776300.475316:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776300.475317:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776300.475319:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776300.475381:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776300.475386:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776300.475390:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776301.371257:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776301.371266:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776301.371269:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776301.371277:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776301.371281:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776301.371285:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776301.371288:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776301.371290:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776301.371292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776301.499275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776301.499283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776301.499290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776301.499291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776301.499295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776301.499297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776301.499458:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776301.499462:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776301.499466:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776302.203232:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307857158, at 4307857216 00000100:00000040:0.0:1775776302.203241:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307857158, at 4307857216 00000100:00000010:1.0:1775776302.203243:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e29c0. 00000100:00000010:0.0:1775776302.203250:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e21e0. 00000100:00000001:3.0:1775776302.395265:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776302.395271:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776302.395273:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776302.395287:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776302.395291:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776302.395295:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776302.395298:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776302.395300:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776302.395303:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776302.523249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776302.523254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776302.523259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776302.523261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776302.523265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776302.523266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776302.523333:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776302.523337:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776302.523340:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776303.419236:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776303.419243:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776303.419245:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775776303.419246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775776303.419249:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776303.419251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776303.419253:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776303.419260:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13186 00000100:00000001:2.0:1775776303.419262:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776303.419269:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00100000:2.0:1775776303.419270:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776303.419275:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1775776303.419277:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:2.0:1775776303.419279:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775776303.419281:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775776303.419284:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1775776303.419287:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776303.419288:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776303.419291:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775776303.419292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776303.419296:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776303.419299:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776303.419303:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776303.419305:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776303.419308:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776303.419311:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776303.419313:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776303.419316:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776303.419318:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776303.419320:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776303.419321:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776303.419323:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776303.419325:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776303.419328:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776303.419329:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776303.419331:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776303.419334:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776303.419335:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776303.419337:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776303.419338:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776303.419346:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775776303.419348:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776303.419351:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776303.419352:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776303.419354:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776303.419356:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776303.419359:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776303.419364:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000001:3.0:1775776303.419368:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776303.419374:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710404224/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776303.419381:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776303.419384:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776303.419387:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710404224/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776303.419403:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776303.419405:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776303.419407:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776303.419408:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776303.419409:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776303.419410:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776303.419412:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13186 00000400:00000200:3.0:1775776303.419415:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776303.419416:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776303.419418:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776303.419419:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776303.419420:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776303.419421:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776303.419423:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776303.419424:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776303.419425:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776303.419426:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776303.419427:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776303.419428:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776303.419429:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776303.419431:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776303.419433:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776303.419434:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776303.419435:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776303.419436:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776303.419438:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776303.419441:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776303.419442:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776303.419447:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776303.419449:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776303.419450:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776303.419453:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:3.0:1775776303.419454:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776303.419457:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776303.419458:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776303.419459:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776303.419460:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000010:3.0:1775776303.419462:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775776303.419465:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000040:1.0:1775776303.419467:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71180 x1862022710404224/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775776303.419468:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776303.419470:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776303.419474:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776303.419476:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776303.419479:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776303.419480:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776303.419482:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776303.419482:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 02000000:00000001:1.0:1775776303.419487:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1775776303.419487:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776303.419490:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776303.419494:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776303.419494:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710404352/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775776303.419498:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710404224:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776303.419505:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1775776303.419508:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1775776303.419510:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1775776303.419510:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776303.419512:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776303.419516:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1775776303.419516:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710404352/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1775776303.419522:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 00000100:00000001:3.0:1775776303.419523:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776303.419525:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776303.419526:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776303.419530:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:1.0:1775776303.419532:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776303.419532:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776303.419535:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:1.0:1775776303.419536:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000100:00000001:3.0:1775776303.419536:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776303.419540:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13186 00000400:00000010:1.0:1775776303.419542:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000400:00000200:3.0:1775776303.419542:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776303.419548:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776303.419549:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000200:1.0:1775776303.419551:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710404224, portal 25 00000100:00000040:3.0:1775776303.419551:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776303.419552:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776303.419554:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776303.419556:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849008512 : -123823860543104 : ffff8f6202c71180) 00000100:00000040:3.0:1775776303.419556:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776303.419559:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776303.419563:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000040:1.0:1775776303.419564:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71180 x1862022710404224/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776328 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775776303.419564:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776303.419567:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776303.419569:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776303.419573:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776303.419577:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000001:1.0:1775776303.419578:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776303.419581:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000040:1.0:1775776303.419582:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776303.419585:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1775776303.419587:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000040:3.0:1775776303.419588:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000200:1.0:1775776303.419591:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710404224, offset 0 00000100:00000001:3.0:1775776303.419591:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776303.419593:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776303.419596:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776303.419597:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776303.419598:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776303.419600:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775776303.419604:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 02000000:00000001:3.0:1775776303.419604:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776303.419606:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776303.419608:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776303.419609:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776303.419610:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776303.419613:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000400:00000200:1.0:1775776303.419614:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000040:3.0:1775776303.419615:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776303.419618:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776303.419619:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776303.419620:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776303.419622:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1775776303.419624:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 02000000:00000001:3.0:1775776303.419624:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776303.419628:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:3.0:1775776303.419631:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776303.419632:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00100000:3.0:1775776303.419636:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710404480/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776303.419639:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:3.0:1775776303.419644:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775776303.419646:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1775776303.419647:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776303.419653:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000040:3.0:1775776303.419654:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710404480/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776303.419663:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453710:453710:256:4294967295] 192.168.201.115@tcp LPNI seq info [453635:453635:8:4294967295] 00000100:00000001:3.0:1775776303.419664:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776303.419669:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776303.419674:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775776303.419676:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776303.419681:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776303.419688:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776303.419693:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13196) 00000400:00000200:1.0:1775776303.419697:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776303.419705:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776303.419710:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776303.419716:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776303.419719:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775776303.419723:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776303.419725:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000100:1.0:1775776303.419728:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776303.419729:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776303.419734:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776303.419737:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776303.419740:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776303.419742:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776303.419746:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776303.419749:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71180 x1862022710404224/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776328 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.419757:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776303.419760:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71180 x1862022710404224/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776328 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.419766:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776303.419768:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776303.419774:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776303.419778:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776303.419780:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307862278 00000100:00000010:1.0:1775776303.419785:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e29c0. 00000100:00000001:1.0:1775776303.419790:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776303.419796:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307862278 00000100:00000010:1.0:1775776303.419799:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e22a0. 00000100:00000001:1.0:1775776303.419801:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776303.419804:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776303.419807:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776303.419809:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776303.419812:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776303/real 1775776303] req@ffff8f6202c71180 x1862022710404224/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776303.419820:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776303.419822:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776303.419823:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776303.419826:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776303.419827:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776303.419830:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776303.419833:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71180 x1862022710404224/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776303.419841:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71180 x1862022710404224/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.419847:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776303.419848:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776303.419851:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776303.419852:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776303.419856:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71180 x1862022710404224/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.419874:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776303.419878:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71180 x1862022710404224/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.419885:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776303.419887:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776303.419889:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776303.419890:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776303.419892:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776303.419894:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776303.419897:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776303.419901:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776303.419903:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776303.419906:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71180 x1862022710404224/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776303.419911:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710404224:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776303.419914:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776303.419915:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776303.419917:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71180 x1862022710404224/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.419921:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776303.419922:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776303.419924:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776303.419926:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776303.419927:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776303.419928:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776303.419929:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776303.419931:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37800. 02000000:00000001:1.0:1775776303.419934:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776303.419935:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776303.419936:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71180. 00000100:00000001:1.0:1775776303.419939:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776303.419939:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776303.419941:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776303.419945:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776303.419946:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776303.419948:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776303.419948:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776303.419949:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776303.419951:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70700 x1862022710404352/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.419955:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776303.419956:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776303.419958:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776303.419959:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776303.419960:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776303.419962:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710404352:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776303.419964:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776303.419965:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776303.419966:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776303.419967:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776303.419969:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776303.419970:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776303.419973:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776303.419974:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776303.419977:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776303.419979:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710404352, portal 10 00000100:00000001:1.0:1775776303.419980:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776303.419981:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849005824 : -123823860545792 : ffff8f6202c70700) 00000100:00000040:1.0:1775776303.419985:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70700 x1862022710404352/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776328 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.419988:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776303.419990:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776303.419991:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776303.419992:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710404352, offset 0 00000400:00000200:1.0:1775776303.419994:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776303.419997:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776303.419999:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776303.420001:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776303.420003:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776303.420006:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776303.420008:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776303.420010:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453711:453711:256:4294967295] 192.168.201.115@tcp LPNI seq info [453636:453636:8:4294967295] 00000400:00000200:1.0:1775776303.420014:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776303.420020:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776303.420025:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776303.420029:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776303.420033:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776303.420036:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775776303.420040:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776303.420042:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000100:1.0:1775776303.420046:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776303.420057:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776303.420060:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776303.420063:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776303.420066:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776303.420071:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776303.420074:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776303.420076:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70700 x1862022710404352/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776328 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.420081:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776303.420083:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70700 x1862022710404352/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776328 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.420086:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776303.420087:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776303.420089:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776303.420092:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776303.420094:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776303.420096:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307858443 00000100:00000001:1.0:1775776303.420097:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307862278 00000100:00000001:1.0:1775776303.420098:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307858443 00000100:00000001:1.0:1775776303.420100:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307862278 00000100:00000001:1.0:1775776303.420101:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776303.420102:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776303.420103:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776303.420105:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776303/real 1775776303] req@ffff8f6202c70700 x1862022710404352/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776303.420114:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776303.420115:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776303.420116:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776303.420118:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776303.420118:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776303.420120:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776303.420121:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70700 x1862022710404352/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776303.420126:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70700 x1862022710404352/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.420129:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776303.420130:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776303.420131:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776303.420132:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776303.420154:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70700 x1862022710404352/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.420166:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776303.420169:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70700 x1862022710404352/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.420175:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776303.420177:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776303.420179:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776303.420180:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776303.420182:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776303.420186:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776303.420189:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776303.420192:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776303.420195:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776303.420199:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70700 x1862022710404352/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776303.420206:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710404352:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776303.420210:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776303.420213:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70a80 x1862022710404480/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.420217:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776303.420217:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776303.420219:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776303.420220:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776303.420221:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776303.420223:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710404480:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776303.420224:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776303.420226:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776303.420227:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776303.420228:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776303.420230:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776303.420232:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776303.420234:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776303.420235:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776303.420237:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776303.420239:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710404480, portal 10 00000100:00000001:1.0:1775776303.420240:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776303.420241:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849006720 : -123823860544896 : ffff8f6202c70a80) 00000100:00000040:1.0:1775776303.420245:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70a80 x1862022710404480/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776328 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.420248:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776303.420250:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776303.420251:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776303.420252:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710404480, offset 0 00000400:00000200:1.0:1775776303.420256:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776303.420259:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776303.420262:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776303.420265:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776303.420267:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776303.420269:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776303.420272:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776303.420274:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453712:453712:256:4294967295] 192.168.201.115@tcp LPNI seq info [453637:453637:8:4294967295] 00000400:00000200:1.0:1775776303.420278:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776303.420284:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776303.420288:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776303.420291:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776303.420295:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776303.420297:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1775776303.420300:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776303.420303:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 6 secs later) 00000800:00000100:1.0:1775776303.420306:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776303.420308:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776303.420310:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776303.420313:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776303.420316:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776303.420318:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776303.420320:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776303.420324:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70a80 x1862022710404480/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776328 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.420330:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776303.420332:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70a80 x1862022710404480/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776328 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.420343:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776303.420344:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776303.420347:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776303.420350:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776303.420353:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776303.420355:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307858444 00000100:00000001:1.0:1775776303.420358:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307862278 00000100:00000001:1.0:1775776303.420360:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307858444 00000100:00000001:1.0:1775776303.420361:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307862278 00000100:00000001:1.0:1775776303.420363:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776303.420365:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776303.420366:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776303.420370:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776303/real 1775776303] req@ffff8f6202c70a80 x1862022710404480/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776303.420377:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776303.420378:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776303.420379:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776303.420381:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776303.420382:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776303.420384:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776303.420386:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70a80 x1862022710404480/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776303.420393:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70a80 x1862022710404480/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.420399:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776303.420400:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776303.420402:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776303.420404:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776303.420407:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70a80 x1862022710404480/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.420412:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776303.420415:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70a80 x1862022710404480/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.420421:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776303.420423:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776303.420424:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776303.420425:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776303.420427:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776303.420428:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776303.420431:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776303.420433:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776303.420435:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776303.420438:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70a80 x1862022710404480/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776303.420444:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710404480:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776303.420447:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776303.420449:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776303.420452:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70700 x1862022710404352/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.420458:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776303.420459:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776303.420460:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776303.420463:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776303.420464:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776303.420465:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776303.420466:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776303.420468:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35000. 02000000:00000001:1.0:1775776303.420470:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776303.420471:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776303.420472:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70700. 00000100:00000001:1.0:1775776303.420474:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776303.420475:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776303.420477:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776303.420479:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70a80 x1862022710404480/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776328 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776303.420490:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776303.420491:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776303.420492:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776303.420494:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776303.420495:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776303.420496:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776303.420498:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776303.420499:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37c00. 02000000:00000001:1.0:1775776303.420501:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776303.420502:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776303.420503:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70a80. 00000100:00000001:1.0:1775776303.420505:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776303.420506:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776303.420508:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776303.420511:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776303.420512:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776303.420514:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776303.420515:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776303.547255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776303.547260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776303.547266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776303.547267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776303.547271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776303.547272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776303.547333:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776303.547336:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776303.547339:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776304.443228:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776304.443234:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776304.443235:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776304.443241:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776304.443245:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776304.443248:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776304.443251:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776304.443253:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776304.443255:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776304.571255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776304.571259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776304.571262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776304.571263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776304.571266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776304.571266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776304.571320:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776304.571322:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776304.571325:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776305.467261:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776305.467272:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776305.467275:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776305.467280:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776305.467283:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776305.467287:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776305.467289:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776305.467292:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776305.467295:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776305.595266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776305.595273:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776305.595276:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776305.595279:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776305.595284:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776305.595286:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776305.595288:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776305.595291:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776305.595294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776305.595296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776305.595300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776305.595301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776305.595364:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776305.595367:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776305.595371:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776306.491253:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776306.491260:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776306.491262:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776306.491267:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776306.491271:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776306.491274:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776306.491277:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776306.491280:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776306.491282:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776306.619260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776306.619265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776306.619271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776306.619272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776306.619275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776306.619276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776306.619351:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776306.619355:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776306.619358:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776307.323234:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307862278, at 4307862336 00000100:00000010:1.0:1775776307.323248:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e22a0. 00000100:00000040:0.0:1775776307.323250:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307862278, at 4307862336 00000100:00000010:0.0:1775776307.323269:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e29c0. 00000100:00000001:1.0:1775776307.515255:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776307.515261:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776307.515263:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776307.515268:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776307.515272:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776307.515276:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776307.515278:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776307.515281:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776307.515283:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776307.643252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776307.643259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776307.643263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776307.643265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776307.643268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776307.643270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776307.643392:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776307.643396:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776307.643400:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776308.539232:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776308.539239:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776308.539241:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776308.539246:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776308.539250:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776308.539253:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776308.539255:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1775776308.539255:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776308.539257:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776308.539262:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776308.539263:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776308.539266:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776308.539268:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776308.539271:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776308.539276:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13191 00000400:00000200:2.0:1775776308.539284:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776308.539288:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776308.539291:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776308.539293:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776308.539295:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776308.539296:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776308.539298:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776308.539300:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776308.539301:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776308.539303:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776308.539305:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776308.539306:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776308.539307:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776308.539309:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776308.539310:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776308.539311:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776308.539313:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776308.539315:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776308.539316:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776308.539318:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776308.539320:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776308.539323:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776308.539324:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776308.539326:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776308.539327:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776308.539335:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775776308.539337:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776308.539340:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776308.539341:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776308.539343:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776308.539345:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776308.539348:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776308.539354:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775776308.539359:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776308.539365:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710404608/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776308.539372:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776308.539375:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776308.539379:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710404608/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776308.539401:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776308.539404:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776308.539408:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776308.539410:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776308.539411:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776308.539414:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776308.539417:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13191 00000400:00000200:2.0:1775776308.539422:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776308.539425:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:1.0:1775776308.539427:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1775776308.539428:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776308.539430:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776308.539433:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776308.539433:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776308.539437:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776308.539440:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344aa00 x1862022710404608/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775776308.539441:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776308.539446:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776308.539450:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776308.539452:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:1.0:1775776308.539454:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1775776308.539455:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775776308.539457:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775776308.539458:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776308.539461:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776308.539462:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1775776308.539462:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776308.539465:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1775776308.539465:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776308.539470:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776308.539470:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776308.539472:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00100000:1.0:1775776308.539476:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710404608:192.168.201.115@tcp:250:kworker.0 00000100:00000001:2.0:1775776308.539476:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776308.539480:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000001:1.0:1775776308.539481:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1775776308.539484:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000001:1.0:1775776308.539485:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1775776308.539486:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776308.539488:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776308.539488:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776308.539491:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1775776308.539494:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:1.0:1775776308.539496:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:2.0:1775776308.539497:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776308.539500:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776308.539500:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776308.539505:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1775776308.539509:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000008:00000001:2.0:1775776308.539510:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776308.539512:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776308.539514:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000100:00000010:2.0:1775776308.539516:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000400:00000010:1.0:1775776308.539519:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000020:00000040:2.0:1775776308.539519:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776308.539520:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776308.539523:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776308.539524:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776308.539525:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:1.0:1775776308.539526:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710404608, portal 25 02000000:00000001:2.0:1775776308.539526:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776308.539531:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776308.539533:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662545920 : -123823047005696 : ffff8f623344aa00) 02000000:00000010:2.0:1775776308.539534:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775776308.539539:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776308.539540:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344aa00 x1862022710404608/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776333 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775776308.539545:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710404736/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776308.539550:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776308.539556:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:2.0:1775776308.539557:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1775776308.539559:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000001:2.0:1775776308.539559:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776308.539563:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710404608, offset 0 00000100:00000040:2.0:1775776308.539566:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710404736/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776308.539572:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:2.0:1775776308.539578:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776308.539580:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776308.539581:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00080000:2.0:1775776308.539586:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776308.539588:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1775776308.539589:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00080000:2.0:1775776308.539590:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775776308.539593:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1775776308.539593:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776308.539596:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13191 00000400:00000200:1.0:1775776308.539599:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:2.0:1775776308.539602:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776308.539604:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776308.539606:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000400:00000200:1.0:1775776308.539609:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00080000:2.0:1775776308.539610:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776308.539615:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776308.539616:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453713:453713:256:4294967295] 192.168.201.115@tcp LPNI seq info [453638:453638:8:4294967295] 00000100:00000001:2.0:1775776308.539617:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776308.539621:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776308.539622:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776308.539624:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776308.539625:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776308.539626:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776308.539628:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000400:00000200:1.0:1775776308.539630:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000001:2.0:1775776308.539632:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776308.539635:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776308.539641:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776308.539645:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:2.0:1775776308.539646:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776308.539651:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776308.539653:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000200:1.0:1775776308.539656:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000020:00000040:2.0:1775776308.539658:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776308.539663:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000800:00000010:1.0:1775776308.539664:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000100:00000001:2.0:1775776308.539666:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776308.539669:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776308.539671:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.0:1775776308.539672:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:2.0:1775776308.539677:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776308.539678:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000100:00000001:2.0:1775776308.539681:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776308.539684:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000200:1.1:1775776308.539685:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000008:00000001:2.0:1775776308.539687:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776308.539689:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000100:00000001:2.0:1775776308.539690:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1775776308.539695:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000010:2.0:1775776308.539695:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000800:00000010:1.0:1775776308.539697:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000020:00000040:2.0:1775776308.539698:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776308.539701:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776308.539702:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776308.539704:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1775776308.539704:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776308.539705:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776308.539709:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776308.539712:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 02000000:00000010:2.0:1775776308.539714:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000400:00000200:1.0:1775776308.539717:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000100:00000001:2.0:1775776308.539717:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776308.539721:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00100000:2.0:1775776308.539724:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710404864/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776308.539728:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1775776308.539729:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776308.539731:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776308.539735:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344aa00 x1862022710404608/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776333 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775776308.539736:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710404864/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776308.539761:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776308.539764:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776308.539765:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776308.539768:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:1.0:1775776308.539784:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344aa00 x1862022710404608/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776333 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775776308.539784:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776308.539793:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776308.539797:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776308.539799:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:2.0:1775776308.539800:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13201) 00000100:00000001:1.0:1775776308.539805:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776308.539809:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776308.539811:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307867398 00000100:00000010:1.0:1775776308.539815:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e22a0. 00000100:00000001:1.0:1775776308.539820:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776308.539825:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307867398 00000100:00000010:1.0:1775776308.539827:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2360. 00000100:00000001:1.0:1775776308.539830:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776308.539832:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776308.539834:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776308.539835:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776308.539838:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776308/real 1775776308] req@ffff8f623344aa00 x1862022710404608/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776308.539844:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776308.539845:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776308.539847:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776308.539849:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776308.539850:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776308.539852:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776308.539854:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344aa00 x1862022710404608/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776308.539863:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344aa00 x1862022710404608/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776308.539867:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776308.539868:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776308.539869:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776308.539870:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776308.539872:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344aa00 x1862022710404608/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776308.539876:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776308.539878:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344aa00 x1862022710404608/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776308.539882:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776308.539884:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776308.539885:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776308.539885:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776308.539887:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776308.539888:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776308.539890:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776308.539892:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776308.539893:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776308.539895:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344aa00 x1862022710404608/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776308.539900:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710404608:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776308.539903:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776308.539905:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776308.539906:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344aa00 x1862022710404608/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776308.539913:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776308.539915:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776308.539916:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776308.539918:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776308.539919:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776308.539920:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776308.539921:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776308.539923:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4d400. 02000000:00000001:1.0:1775776308.539925:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776308.539926:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776308.539927:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344aa00. 00000100:00000001:1.0:1775776308.539931:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776308.539931:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776308.539933:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776308.539937:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776308.539938:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776308.539940:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776308.539940:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776308.539941:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776308.539943:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233448380 x1862022710404736/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776308.539947:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776308.539948:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776308.539949:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776308.539950:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776308.539952:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776308.539953:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233448380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710404736:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776308.539955:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776308.539956:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776308.539957:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776308.539958:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776308.539959:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776308.539961:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776308.539963:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776308.539964:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776308.539966:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776308.539968:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710404736, portal 10 00000100:00000001:1.0:1775776308.539969:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776308.539970:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662536064 : -123823047015552 : ffff8f6233448380) 00000100:00000040:1.0:1775776308.539974:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233448380 x1862022710404736/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776333 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776308.539978:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776308.539979:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776308.539980:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776308.539982:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710404736, offset 0 00000400:00000200:1.0:1775776308.539984:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776308.539986:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776308.539988:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776308.539990:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776308.539991:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776308.539993:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776308.539994:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776308.539996:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453714:453714:256:4294967295] 192.168.201.115@tcp LPNI seq info [453639:453639:8:4294967295] 00000400:00000200:1.0:1775776308.539998:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776308.540002:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776308.540004:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776308.540006:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776308.540008:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776308.540010:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775776308.540013:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776308.540014:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1775776308.540016:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776308.540017:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776308.540019:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776308.540021:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776308.540022:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776308.540023:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776308.540025:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776308.540027:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233448380 x1862022710404736/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776333 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776308.540032:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776308.540033:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448380 x1862022710404736/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776333 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776308.540037:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776308.540038:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776308.540040:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776308.540042:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776308.540043:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776308.540045:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307863563 00000100:00000001:1.0:1775776308.540046:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307867398 00000100:00000001:1.0:1775776308.540047:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307863563 00000100:00000001:1.0:1775776308.540049:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307867398 00000100:00000001:1.0:1775776308.540050:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776308.540051:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776308.540052:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776308.540054:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776308/real 1775776308] req@ffff8f6233448380 x1862022710404736/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776308.540058:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776308.540058:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776308.540059:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776308.540060:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776308.540061:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776308.540062:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776308.540064:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233448380 x1862022710404736/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776308.540068:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233448380 x1862022710404736/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776308.540075:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776308.540076:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776308.540077:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776308.540078:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776308.540080:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233448380 x1862022710404736/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776308.540083:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776308.540085:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233448380 x1862022710404736/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776308.540088:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776308.540090:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776308.540090:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776308.540091:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776308.540092:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776308.540095:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776308.540096:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776308.540098:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776308.540099:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776308.540101:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233448380 x1862022710404736/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776308.540105:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233448380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710404736:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776308.540107:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776308.540109:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449f80 x1862022710404864/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776308.540113:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776308.540113:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776308.540115:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776308.540116:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776308.540117:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776308.540119:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710404864:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776308.540121:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776308.540122:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776308.540122:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776308.540123:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776308.540125:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776308.540126:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776308.540128:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776308.540129:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776308.540131:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776308.540148:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710404864, portal 10 00000100:00000001:1.0:1775776308.540150:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776308.540151:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662543232 : -123823047008384 : ffff8f6233449f80) 00000100:00000040:1.0:1775776308.540154:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449f80 x1862022710404864/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776333 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776308.540158:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776308.540159:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776308.540161:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776308.540162:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710404864, offset 0 00000400:00000200:1.0:1775776308.540166:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776308.540169:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776308.540172:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776308.540174:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776308.540177:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776308.540179:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776308.540180:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776308.540183:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453715:453715:256:4294967295] 192.168.201.115@tcp LPNI seq info [453640:453640:8:4294967295] 00000400:00000200:1.0:1775776308.540192:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776308.540198:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776308.540202:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776308.540206:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776308.540210:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776308.540212:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1775776308.540214:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776308.540216:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1775776308.540218:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776308.540219:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:1.0:1775776308.540221:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776308.540223:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776308.540225:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:1.0:1775776308.540226:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:1.0:1775776308.540228:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776308.540230:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449f80 x1862022710404864/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776333 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776308.540234:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776308.540236:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449f80 x1862022710404864/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776333 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776308.540240:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776308.540241:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776308.540243:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776308.540244:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776308.540246:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776308.540247:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307863564 00000100:00000001:1.0:1775776308.540249:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307867398 00000100:00000001:1.0:1775776308.540250:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307863564 00000100:00000001:1.0:1775776308.540251:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307867398 00000100:00000001:1.0:1775776308.540253:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776308.540254:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776308.540255:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776308.540257:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776308/real 1775776308] req@ffff8f6233449f80 x1862022710404864/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776308.540262:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776308.540263:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776308.540264:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776308.540265:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776308.540266:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776308.540267:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776308.540268:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449f80 x1862022710404864/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776308.540273:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449f80 x1862022710404864/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776308.540276:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776308.540277:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776308.540278:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776308.540279:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776308.540280:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449f80 x1862022710404864/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776308.540284:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776308.540285:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449f80 x1862022710404864/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776308.540289:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776308.540290:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776308.540291:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776308.540292:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776308.540293:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776308.540294:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776308.540295:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776308.540296:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776308.540297:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776308.540300:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449f80 x1862022710404864/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776308.540304:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710404864:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776308.540306:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776308.540307:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776308.540309:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233448380 x1862022710404736/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776308.540313:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776308.540314:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776308.540316:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776308.540318:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776308.540319:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776308.540320:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776308.540321:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776308.540323:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4e800. 02000000:00000001:1.0:1775776308.540325:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776308.540326:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776308.540328:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233448380. 00000100:00000001:1.0:1775776308.540330:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776308.540331:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776308.540332:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776308.540335:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449f80 x1862022710404864/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776333 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776308.540342:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776308.540343:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776308.540344:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776308.540346:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776308.540347:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776308.540348:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776308.540350:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776308.540351:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4c000. 02000000:00000001:1.0:1775776308.540353:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776308.540354:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776308.540356:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449f80. 00000100:00000001:1.0:1775776308.540358:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776308.540359:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776308.540360:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776308.540363:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776308.540365:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776308.540366:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776308.540368:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776308.667261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776308.667266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776308.667270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776308.667271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776308.667275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776308.667275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776308.667346:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776308.667351:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776308.667353:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776309.563993:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776309.563999:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776309.564001:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776309.564006:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776309.564009:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776309.564013:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776309.564015:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776309.564017:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776309.564019:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776309.691273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776309.691279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776309.691285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776309.691287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776309.691291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776309.691292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776309.691350:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776309.691353:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776309.691356:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776310.587244:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776310.587249:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776310.587250:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776310.587254:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776310.587256:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776310.587260:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776310.587262:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776310.587263:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776310.587265:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776310.715255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776310.715261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776310.715266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776310.715270:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776310.715272:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776310.715275:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776310.715278:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776310.715280:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776310.715282:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776310.715284:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776310.715288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776310.715289:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776310.715290:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776310.715291:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776310.715293:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776310.715294:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776310.715295:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776310.715296:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776310.715402:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776310.715405:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776310.715416:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776311.611248:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776311.611255:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776311.611257:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776311.611262:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776311.611265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776311.611269:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776311.611272:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776311.611274:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776311.611277:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776311.739366:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776311.739371:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776311.739374:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776311.739375:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776311.739377:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776311.739378:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776311.739430:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776311.739433:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776311.739435:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776312.443244:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307867398, at 4307867456 00000100:00000010:1.0:1775776312.443260:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2360. 00000100:00000040:0.0:1775776312.443261:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307867398, at 4307867456 00000100:00000010:0.0:1775776312.443286:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e22a0. 00000100:00000001:1.0:1775776312.635252:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776312.635259:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776312.635261:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776312.635266:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776312.635269:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776312.635273:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776312.635275:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776312.635277:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776312.635280:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776312.763257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776312.763263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776312.763269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776312.763270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776312.763274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776312.763276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776312.763326:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776312.763330:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776312.763333:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775776313.659362:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776313.659371:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776313.659373:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776313.659377:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776313.659381:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776313.659388:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13196 00000400:00000200:2.0:1775776313.659397:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776313.659404:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776313.659407:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776313.659410:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776313.659414:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776313.659416:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776313.659418:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776313.659422:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776313.659424:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776313.659427:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776313.659429:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776313.659431:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776313.659433:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776313.659436:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776313.659438:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776313.659440:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776313.659441:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776313.659443:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776313.659447:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776313.659449:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776313.659452:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776313.659455:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776313.659458:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776313.659460:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776313.659462:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776313.659470:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775776313.659473:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776313.659477:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776313.659478:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776313.659481:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776313.659483:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776313.659486:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776313.659493:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775776313.659499:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776313.659507:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710404992/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776313.659516:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776313.659520:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776313.659523:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710404992/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776313.659544:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776313.659547:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776313.659550:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776313.659551:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776313.659553:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776313.659555:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776313.659557:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13196 00000400:00000200:2.0:1775776313.659560:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776313.659563:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776313.659565:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776313.659569:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776313.659571:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776313.659573:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776313.659575:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776313.659577:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776313.659578:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776313.659579:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776313.659580:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776313.659582:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776313.659584:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776313.659587:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776313.659589:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776313.659590:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776313.659592:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776313.659594:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776313.659596:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776313.659598:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776313.659600:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776313.659604:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776313.659606:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776313.659608:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776313.659612:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776313.659614:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776313.659616:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776313.659620:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775776313.659623:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776313.659625:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776313.659626:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776313.659627:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776313.659629:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776313.659630:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776313.659633:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775776313.659636:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776313.659641:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710405120/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776313.659648:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776313.659650:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776313.659653:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710405120/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776313.659660:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776313.659663:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776313.659666:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776313.659667:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776313.659668:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776313.659671:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776313.659673:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13196 00000400:00000200:2.0:1775776313.659676:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776313.659679:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776313.659698:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776313.659700:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776313.659703:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776313.659704:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776313.659706:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776313.659708:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776313.659710:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776313.659712:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776313.659713:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776313.659715:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776313.659717:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776313.659720:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776313.659722:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776313.659724:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776313.659726:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776313.659728:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776313.659730:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776313.659732:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776313.659734:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776313.659737:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776313.659738:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776313.659740:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776313.659742:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776313.659744:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776313.659745:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776313.659748:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775776313.659750:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776313.659752:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776313.659754:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776313.659755:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776313.659756:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776313.659758:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776313.659761:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775776313.659764:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776313.659768:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710405248/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776313.659775:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776313.659777:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776313.659780:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710405248/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776313.659788:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776313.659791:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776313.659795:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776313.659797:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776313.659801:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776313.659805:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13206) 00000100:00000001:1.0:1775776313.661364:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776313.661370:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776313.661372:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776313.661380:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344a680 x1862022710404992/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776313.661390:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776313.661392:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776313.661395:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1775776313.661395:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 02000000:00000001:1.0:1775776313.661398:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776313.661399:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:1.0:1775776313.661400:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776313.661401:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776313.661403:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710404992:192.168.201.115@tcp:250:kworker.0 00000100:00100000:3.0:1775776313.661404:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776313.661406:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776313.661408:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1775776313.661409:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775776313.661411:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1775776313.661411:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776313.661412:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1775776313.661413:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776313.661415:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775776313.661415:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776313.661421:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776313.661424:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776313.661431:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776313.661435:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776313.661442:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776313.661449:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710404992, portal 25 00000100:00000001:1.0:1775776313.661451:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776313.661452:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662545024 : -123823047006592 : ffff8f623344a680) 00000100:00000040:1.0:1775776313.661457:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344a680 x1862022710404992/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776338 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776313.661464:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776313.661467:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776313.661469:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3198. 00000100:00000200:1.0:1775776313.661472:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710404992, offset 0 00000400:00000200:1.0:1775776313.661478:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776313.661486:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776313.661493:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776313.661498:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776313.661502:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776313.661506:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776313.661508:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776313.661512:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453716:453716:256:4294967295] 192.168.201.115@tcp LPNI seq info [453641:453641:8:4294967295] 00000400:00000200:1.0:1775776313.661516:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776313.661526:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776313.661532:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776313.661537:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230900. 00000800:00000200:1.0:1775776313.661543:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776313.661547:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:1.0:1775776313.661565:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776313.661568:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776313.661570:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307872520 00000100:00000010:1.0:1775776313.661574:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2360. 00000100:00000001:1.0:1775776313.661579:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776313.661583:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307872520 00000100:00000010:1.0:1775776313.661585:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2420. 00000100:00000001:1.0:1775776313.661587:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776313.661589:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776313.661591:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776313.661593:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776313.661597:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449f80 x1862022710405120/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776313.661602:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776313.661603:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776313.661605:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776313.661606:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776313.661608:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776313.661610:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710405120:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776313.661613:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776313.661615:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776313.661617:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776313.661618:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776313.661621:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776313.661622:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776313.661626:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776313.661630:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fe000. 00000400:00000010:1.0:1775776313.661634:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776313.661637:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710405120, portal 10 00000100:00000001:1.0:1775776313.661639:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776313.661641:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662543232 : -123823047008384 : ffff8f6233449f80) 00000100:00000040:1.0:1775776313.661648:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449f80 x1862022710405120/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776338 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776313.661654:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776313.661657:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776313.661661:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f33b8. 00000100:00000200:1.0:1775776313.661664:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710405120, offset 0 00000400:00000200:1.0:1775776313.661668:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776313.661673:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776313.661676:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776313.661679:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776313.661697:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776313.661700:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776313.661702:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776313.661705:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453717:453717:255:4294967295] 192.168.201.115@tcp LPNI seq info [453642:453642:7:4294967295] 00000400:00000200:1.0:1775776313.661709:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776313.661714:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776313.661717:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776313.661720:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230600. 00000800:00000200:1.0:1775776313.661723:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776313.661725:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:1.0:1775776313.661727:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776313.661731:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776313.661733:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776313.661734:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307868685 00000100:00000001:1.0:1775776313.661736:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307872520 00000100:00000001:1.0:1775776313.661737:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307868685 00000100:00000001:1.0:1775776313.661739:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307872520 00000100:00000001:1.0:1775776313.661740:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776313.661741:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776313.661743:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776313.661746:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233448380 x1862022710405248/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776313.661752:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776313.661753:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776313.661755:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776313.661757:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776313.661759:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776313.661762:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233448380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710405248:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776313.661766:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776313.661768:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776313.661770:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776313.661771:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776313.661774:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776313.661776:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776313.661780:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776313.661782:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776313.661785:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776313.661788:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710405248, portal 10 00000100:00000001:1.0:1775776313.661790:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776313.661791:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662536064 : -123823047015552 : ffff8f6233448380) 00000100:00000040:1.0:1775776313.661796:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233448380 x1862022710405248/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776338 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776313.661803:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776313.661805:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776313.661807:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3cc0. 00000100:00000200:1.0:1775776313.661809:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710405248, offset 0 00000400:00000200:1.0:1775776313.661813:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776313.661817:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776313.661820:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776313.661823:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776313.661825:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776313.661827:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776313.661830:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776313.661832:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453718:453718:254:4294967295] 192.168.201.115@tcp LPNI seq info [453643:453643:6:4294967295] 00000400:00000200:1.0:1775776313.661836:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776313.661842:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776313.661846:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776313.661849:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230400. 00000800:00000200:1.0:1775776313.661852:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776313.661856:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:1.0:1775776313.661858:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776313.661861:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776313.661864:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776313.661866:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307868685 00000100:00000001:1.0:1775776313.661868:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307872520 00000100:00000001:1.0:1775776313.661869:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307868685 00000100:00000001:1.0:1775776313.661872:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307872520 00000100:00000001:1.0:1775776313.661873:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776313.661875:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776313.661877:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776313.661878:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776313.661884:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776313.661885:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775776313.661886:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776313.661887:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776313.661889:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776313.661890:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776313.787246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776313.787251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776313.787256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776313.787258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776313.787261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776313.787263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776313.787432:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776313.787435:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776313.787438:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776314.683231:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776314.683236:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776314.683237:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776314.683241:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776314.683244:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776314.683246:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776314.683248:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776314.683250:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776314.683251:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776314.811214:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776314.811219:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776314.811222:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776314.811223:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776314.811225:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776314.811226:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776314.811280:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776314.811284:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776314.811287:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776315.707253:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776315.707262:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776315.707264:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776315.707270:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776315.707275:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776315.707279:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776315.707282:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776315.707284:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776315.707287:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776315.835307:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776315.835315:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776315.835317:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776315.835320:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776315.835324:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776315.835326:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776315.835328:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776315.835329:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776315.835332:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776315.835333:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776315.835336:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776315.835337:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776315.835417:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776315.835424:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776315.835429:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776316.731242:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776316.731249:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776316.731251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776316.731258:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776316.731262:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776316.731266:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776316.731269:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776316.731272:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776316.731275:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1775776316.731365:0:2402:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:0.0:1775776316.731395:0:2402:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1775776316.731405:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776316.731408:0:2402:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:0.0:1775776316.731412:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776316.731416:0:2402:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:0.0:1775776316.731422:0:2402:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:0.0:1775776316.731425:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230900. 00000400:00000200:0.0:1775776316.731430:0:2402:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776316.731436:0:2402:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776316.731445:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3198 00000400:00000010:0.0:1775776316.731448:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3198. 00000100:00000001:0.0:1775776316.731455:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776316.731462:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f623344a680 x1862022710404992/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776338 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776316.731490:0:2402:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776316.731494:0:2402:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344a680 x1862022710404992/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776338 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776316.731502:0:2402:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776316.731503:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775776316.731513:0:2402:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:0.0:1775776316.731515:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230600. 00000400:00000200:0.0:1775776316.731518:0:2402:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776316.731522:0:2402:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776316.731525:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f33b8 00000400:00000010:0.0:1775776316.731526:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f33b8. 00000100:00000001:0.0:1775776316.731529:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776316.731533:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6233449f80 x1862022710405120/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776338 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776316.731540:0:2402:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776316.731543:0:2402:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449f80 x1862022710405120/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776338 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776316.731548:0:2402:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776316.731549:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1775776316.731553:0:2402:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:0.0:1775776316.731556:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230400. 00000400:00000200:0.0:1775776316.731558:0:2402:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1775776316.731561:0:2402:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776316.731563:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3cc0 00000400:00000010:0.0:1775776316.731565:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3cc0. 00000100:00000001:0.0:1775776316.731567:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776316.731570:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6233448380 x1862022710405248/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776338 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776316.731577:0:2402:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776316.731579:0:2402:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448380 x1862022710405248/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776338 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776316.731585:0:2402:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776316.731586:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776316.731730:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776316.731734:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776316.731740:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776316.731748:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776313/real 1775776316] req@ffff8f623344a680 x1862022710404992/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776316.731759:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776316.731761:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776316.731765:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776316.731768:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776316.731770:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776316.731773:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776316.731777:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344a680 x1862022710404992/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776316.731790:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344a680 x1862022710404992/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776316.731795:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776316.731797:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776316.731800:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776316.731802:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776316.731805:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344a680 x1862022710404992/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776316.731811:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776316.731815:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344a680 x1862022710404992/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776316.731821:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776316.731824:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776316.731826:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776316.731827:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776316.731829:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776316.731831:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776316.731834:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776316.731839:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776316.731841:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776316.731845:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344a680 x1862022710404992/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776316.731852:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710404992:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776316.731856:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776316.731859:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776313/real 1775776316] req@ffff8f6233449f80 x1862022710405120/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776316.731871:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776316.731872:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776316.731874:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fe000. 00000400:00000200:1.0:1775776316.731877:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776316.731878:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776316.731880:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776316.731883:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449f80 x1862022710405120/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776316.731891:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449f80 x1862022710405120/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776316.731898:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776316.731899:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776316.731901:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776316.731902:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776316.731905:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449f80 x1862022710405120/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776316.731911:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776316.731915:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449f80 x1862022710405120/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776316.731921:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776316.731923:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776316.731924:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776316.731926:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776316.731927:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776316.731932:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776316.731935:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776316.731938:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776316.731941:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776316.731946:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449f80 x1862022710405120/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776316.731954:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710405120:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776316.731958:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776316.731962:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776313/real 1775776316] req@ffff8f6233448380 x1862022710405248/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776316.731970:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776316.731971:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776316.731973:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776316.731975:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776316.731977:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776316.731979:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776316.731981:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233448380 x1862022710405248/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776316.731989:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233448380 x1862022710405248/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776316.731996:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776316.731997:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776316.731999:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776316.732000:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776316.732004:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233448380 x1862022710405248/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776316.732011:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776316.732015:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233448380 x1862022710405248/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776316.732021:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776316.732023:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776316.732026:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776316.732027:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776316.732028:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776316.732031:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776316.732033:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776316.732037:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776316.732039:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776316.732043:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233448380 x1862022710405248/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776316.732051:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233448380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710405248:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776316.732055:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776316.732057:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776316.732059:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344a680 x1862022710404992/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776316.732065:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776316.732068:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776316.732071:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776316.732074:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776316.732076:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776316.732078:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776316.732080:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776316.732083:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4e000. 02000000:00000001:1.0:1775776316.732086:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776316.732088:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776316.732089:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344a680. 00000100:00000001:1.0:1775776316.732095:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776316.732096:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776316.732097:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776316.732100:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449f80 x1862022710405120/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776316.732106:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776316.732107:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776316.732109:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776316.732111:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776316.732112:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776316.732113:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776316.732115:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776316.732116:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4f000. 02000000:00000001:1.0:1775776316.732118:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776316.732119:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776316.732120:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449f80. 00000100:00000001:1.0:1775776316.732122:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776316.732123:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776316.732125:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776316.732127:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233448380 x1862022710405248/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776338 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776316.732156:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776316.732158:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776316.732159:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776316.732162:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776316.732164:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776316.732166:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776316.732168:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776316.732170:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4ec00. 02000000:00000001:1.0:1775776316.732173:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776316.732174:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776316.732175:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233448380. 00000100:00000001:1.0:1775776316.732178:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776316.732179:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776316.732182:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776316.732190:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776316.732192:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776316.732194:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776316.732196:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776316.859264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776316.859270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776316.859275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776316.859276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776316.859279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776316.859280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776316.859341:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776316.859344:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776316.859346:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776317.563219:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307872520, at 4307872576 00000100:00000040:0.0:1775776317.563230:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307872520, at 4307872576 00000100:00000010:1.0:1775776317.563230:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2420. 00000100:00000010:0.0:1775776317.563238:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2360. 00000100:00000001:3.0:1775776317.755282:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776317.755291:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776317.755293:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776317.755300:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776317.755304:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776317.755309:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776317.755312:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776317.755314:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776317.755317:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776317.883298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776317.883310:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776317.883315:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776317.883316:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776317.883319:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776317.883320:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776317.883378:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776317.883382:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776317.883385:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776318.779263:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776318.779271:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776318.779274:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776318.779278:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776318.779283:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776318.779291:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13201 00000400:00000200:3.0:1775776318.779301:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776318.779308:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776318.779312:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775776318.779313:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000040:3.0:1775776318.779315:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776318.779321:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775776318.779323:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:3.0:1775776318.779327:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776318.779328:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776318.779335:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00100000:2.0:1775776318.779336:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776318.779341:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1775776318.779344:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:2.0:1775776318.779345:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776318.779349:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1775776318.779349:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000100:00000001:2.0:1775776318.779352:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776318.779356:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775776318.779356:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776318.779363:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776318.779367:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776318.779371:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776318.779377:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776318.779380:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776318.779383:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776318.779385:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776318.779389:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776318.779392:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776318.779395:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776318.779398:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776318.779402:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776318.779405:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776318.779408:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776318.779410:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776318.779420:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775776318.779424:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776318.779428:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776318.779430:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776318.779433:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776318.779436:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776318.779441:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776318.779451:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:3.0:1775776318.779459:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776318.779470:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710405376/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776318.779482:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776318.779487:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776318.779492:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710405376/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776318.779528:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776318.779531:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776318.779535:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776318.779536:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776318.779537:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776318.779541:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776318.779543:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13201 00000400:00000200:3.0:1775776318.779548:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776318.779551:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776318.779553:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776318.779556:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776318.779557:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776318.779559:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776318.779561:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776318.779563:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776318.779564:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776318.779566:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776318.779568:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776318.779570:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776318.779572:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776318.779575:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776318.779577:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776318.779579:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776318.779580:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776318.779582:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776318.779585:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776318.779587:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776318.779588:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776318.779604:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776318.779606:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776318.779608:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776318.779616:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776318.779618:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776318.779620:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776318.779624:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775776318.779626:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776318.779628:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776318.779629:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776318.779631:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776318.779632:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776318.779634:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776318.779638:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000001:3.0:1775776318.779640:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776318.779646:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710405504/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776318.779652:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776318.779654:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776318.779657:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710405504/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776318.779663:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776318.779665:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776318.779669:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776318.779670:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776318.779672:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776318.779674:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776318.779677:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13201 00000400:00000200:3.0:1775776318.779680:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776318.779683:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776318.779686:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776318.779688:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776318.779690:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776318.779691:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776318.779692:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776318.779694:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776318.779695:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776318.779697:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776318.779699:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776318.779700:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776318.779702:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776318.779705:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776318.779707:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776318.779708:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776318.779710:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776318.779711:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776318.779713:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776318.779715:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776318.779717:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776318.779719:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776318.779721:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776318.779723:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776318.779724:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776318.779726:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776318.779728:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776318.779731:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775776318.779733:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776318.779736:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776318.779737:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776318.779739:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776318.779740:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776318.779741:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776318.779745:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775776318.779748:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776318.779752:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710405632/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776318.779757:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776318.779759:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776318.779762:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710405632/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776318.779769:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776318.779771:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776318.779774:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776318.779777:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776318.779780:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776318.779785:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13211) 00000100:00000001:1.0:1775776318.779960:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776318.779965:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776318.779968:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776318.779975:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71500 x1862022710405376/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.779983:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776318.779985:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776318.779990:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776318.779992:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776318.779995:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776318.779999:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710405376:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776318.780004:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776318.780010:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776318.780013:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776318.780016:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776318.780021:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776318.780023:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776318.780029:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776318.780034:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776318.780042:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776318.780052:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710405376, portal 25 00000100:00000001:1.0:1775776318.780055:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776318.780057:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849009408 : -123823860542208 : ffff8f6202c71500) 00000100:00000040:1.0:1775776318.780063:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71500 x1862022710405376/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776343 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.780071:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776318.780075:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776318.780077:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776318.780081:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710405376, offset 0 00000400:00000200:1.0:1775776318.780089:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776318.780100:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776318.780106:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776318.780112:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776318.780117:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776318.780122:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776318.780124:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776318.780128:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453719:453719:256:4294967295] 192.168.201.115@tcp LPNI seq info [453644:453644:8:4294967295] 00000400:00000200:1.0:1775776318.780175:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776318.780188:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776318.780195:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776318.780202:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776318.780210:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776318.780214:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775776318.780220:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776318.780223:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775776318.780228:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776318.780231:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776318.780237:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776318.780241:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776318.780247:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776318.780249:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776318.780255:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776318.780259:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71500 x1862022710405376/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776343 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.780268:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776318.780272:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71500 x1862022710405376/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776343 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.780278:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776318.780280:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776318.780286:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776318.780290:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776318.780293:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307877639 00000100:00000010:1.0:1775776318.780299:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2420. 00000100:00000001:1.0:1775776318.780307:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776318.780315:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307877639 00000100:00000010:1.0:1775776318.780318:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2600. 00000100:00000001:1.0:1775776318.780321:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776318.780324:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776318.780326:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776318.780329:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776318.780334:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776318/real 1775776318] req@ffff8f6202c71500 x1862022710405376/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776318.780344:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776318.780346:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776318.780348:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776318.780351:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776318.780353:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776318.780355:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776318.780359:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71500 x1862022710405376/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776318.780367:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71500 x1862022710405376/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.780375:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776318.780377:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776318.780379:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776318.780381:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776318.780385:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71500 x1862022710405376/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.780392:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776318.780397:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71500 x1862022710405376/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.780404:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776318.780408:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776318.780409:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776318.780410:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776318.780413:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776318.780417:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776318.780420:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776318.780424:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776318.780426:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776318.780429:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71500 x1862022710405376/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776318.780436:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710405376:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776318.780441:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776318.780443:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70380 x1862022710405504/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.780449:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776318.780450:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776318.780452:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776318.780454:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776318.780456:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776318.780459:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710405504:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776318.780461:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776318.780463:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776318.780465:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776318.780467:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776318.780471:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776318.780473:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776318.780477:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776318.780479:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776318.780483:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776318.780486:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710405504, portal 10 00000100:00000001:1.0:1775776318.780489:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776318.780490:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004928 : -123823860546688 : ffff8f6202c70380) 00000100:00000040:1.0:1775776318.780496:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70380 x1862022710405504/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776343 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.780502:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776318.780505:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776318.780507:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776318.780508:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710405504, offset 0 00000400:00000200:1.0:1775776318.780512:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776318.780515:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776318.780519:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776318.780521:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776318.780524:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776318.780526:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776318.780528:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776318.780530:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453720:453720:256:4294967295] 192.168.201.115@tcp LPNI seq info [453645:453645:8:4294967295] 00000400:00000200:1.0:1775776318.780534:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776318.780539:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776318.780543:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776318.780547:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776318.780550:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776318.780553:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775776318.780557:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776318.780559:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775776318.780562:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776318.780564:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776318.780566:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776318.780569:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776318.780572:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776318.780574:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776318.780576:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776318.780579:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70380 x1862022710405504/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776343 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.780623:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776318.780627:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70380 x1862022710405504/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776343 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.780633:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776318.780634:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776318.780638:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776318.780642:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776318.780645:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776318.780647:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307873804 00000100:00000001:1.0:1775776318.780650:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307877639 00000100:00000001:1.0:1775776318.780652:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307873804 00000100:00000001:1.0:1775776318.780654:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307877639 00000100:00000001:1.0:1775776318.780656:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776318.780658:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776318.780660:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776318.780663:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776318/real 1775776318] req@ffff8f6202c70380 x1862022710405504/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776318.780669:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776318.780670:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776318.780672:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776318.780674:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776318.780675:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776318.780678:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776318.780680:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70380 x1862022710405504/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776318.780688:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70380 x1862022710405504/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.780695:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776318.780697:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776318.780699:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776318.780700:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776318.780703:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70380 x1862022710405504/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.780711:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776318.780715:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70380 x1862022710405504/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.780722:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776318.780725:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776318.780726:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776318.780728:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776318.780729:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776318.780735:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776318.780738:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776318.780742:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776318.780744:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776318.780747:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70380 x1862022710405504/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776318.780754:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710405504:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776318.780758:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776318.780760:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73480 x1862022710405632/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.780765:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776318.780767:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776318.780768:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776318.780770:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776318.780771:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776318.780774:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710405632:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776318.780777:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776318.780778:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776318.780780:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776318.780781:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776318.780785:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776318.780787:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776318.780790:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776318.780792:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776318.780795:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776318.780798:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710405632, portal 10 00000100:00000001:1.0:1775776318.780799:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776318.780801:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849017472 : -123823860534144 : ffff8f6202c73480) 00000100:00000040:1.0:1775776318.780806:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73480 x1862022710405632/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776343 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.780810:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776318.780812:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776318.780814:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776318.780816:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710405632, offset 0 00000400:00000200:1.0:1775776318.780819:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776318.780822:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776318.780826:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776318.780828:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776318.780830:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776318.780832:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776318.780834:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776318.780837:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453721:453721:256:4294967295] 192.168.201.115@tcp LPNI seq info [453646:453646:8:4294967295] 00000400:00000200:1.0:1775776318.780840:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776318.780845:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776318.780848:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776318.780851:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776318.780855:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776318.780857:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775776318.780860:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776318.780863:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775776318.780866:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776318.780868:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776318.780871:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776318.780873:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776318.780876:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776318.780877:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776318.780879:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776318.780882:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73480 x1862022710405632/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776343 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.780888:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776318.780891:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73480 x1862022710405632/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776343 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.780897:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776318.780898:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776318.780900:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776318.780903:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776318.780906:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776318.780907:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307873804 00000100:00000001:1.0:1775776318.780910:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307877639 00000100:00000001:1.0:1775776318.780911:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307873804 00000100:00000001:1.0:1775776318.780914:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307877639 00000100:00000001:1.0:1775776318.780915:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776318.780917:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776318.780918:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776318.780921:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776318/real 1775776318] req@ffff8f6202c73480 x1862022710405632/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776318.780929:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776318.780930:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776318.780932:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776318.780934:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776318.780936:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776318.780938:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776318.780941:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73480 x1862022710405632/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776318.780947:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73480 x1862022710405632/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.780952:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776318.780954:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776318.780955:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776318.780956:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776318.780959:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73480 x1862022710405632/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.780965:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776318.780968:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73480 x1862022710405632/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.780974:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776318.780976:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776318.780977:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776318.780979:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776318.780980:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776318.780982:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776318.780984:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776318.780987:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776318.780989:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776318.780993:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73480 x1862022710405632/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776318.781004:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710405632:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776318.781008:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776318.781010:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776318.781012:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71500 x1862022710405376/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.781018:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776318.781021:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776318.781024:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776318.781027:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776318.781029:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776318.781031:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776318.781033:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776318.781036:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35400. 02000000:00000001:1.0:1775776318.781040:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776318.781042:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776318.781044:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71500. 00000100:00000001:1.0:1775776318.781047:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776318.781049:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776318.781050:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776318.781053:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70380 x1862022710405504/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.781059:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776318.781060:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776318.781062:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776318.781064:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776318.781065:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776318.781066:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776318.781068:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776318.781069:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35800. 02000000:00000001:1.0:1775776318.781071:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776318.781072:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776318.781073:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70380. 00000100:00000001:1.0:1775776318.781076:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776318.781077:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776318.781079:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776318.781081:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73480 x1862022710405632/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776343 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776318.781087:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776318.781088:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776318.781089:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776318.781091:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776318.781092:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776318.781093:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776318.781095:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776318.781096:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37400. 02000000:00000001:1.0:1775776318.781098:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776318.781099:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776318.781100:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73480. 00000100:00000001:1.0:1775776318.781102:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776318.781103:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776318.781106:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776318.781115:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776318.781117:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776318.781119:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776318.781121:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776318.907237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776318.907242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776318.907246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776318.907247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776318.907250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776318.907250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776318.907328:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776318.907333:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776318.907339:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776319.803319:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776319.803326:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776319.803327:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776319.803333:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776319.803337:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776319.803341:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776319.803343:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776319.803345:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776319.803348:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776319.931259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776319.931264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776319.931270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776319.931271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776319.931275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776319.931276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776319.931330:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776319.931333:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776319.931335:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776320.827242:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776320.827247:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776320.827248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776320.827252:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776320.827254:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776320.827257:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776320.827259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776320.827262:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776320.827263:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776320.955234:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776320.955240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776320.955246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776320.955249:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776320.955251:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776320.955255:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776320.955258:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776320.955260:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776320.955262:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776320.955264:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776320.955267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776320.955268:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776320.955269:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776320.955270:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776320.955271:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776320.955272:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776320.955273:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776320.955274:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776320.955323:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776320.955327:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776320.955329:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776321.851236:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776321.851242:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776321.851244:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776321.851249:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776321.851252:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776321.851255:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776321.851258:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776321.851260:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776321.851263:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776321.979228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776321.979231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776321.979234:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776321.979235:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776321.979238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776321.979239:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776321.979292:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776321.979296:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776321.979299:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776322.683259:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307877639, at 4307877696 00000100:00000040:0.0:1775776322.683281:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307877639, at 4307877696 00000100:00000010:1.0:1775776322.683282:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2600. 00000100:00000010:0.0:1775776322.683291:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2420. 00000100:00000001:2.0:1775776322.875272:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776322.875278:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776322.875281:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776322.875286:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776322.875289:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776322.875292:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776322.875295:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776322.875297:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776322.875300:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776323.003284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776323.003293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776323.003300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776323.003301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776323.003305:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776323.003306:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776323.003363:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776323.003366:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776323.003369:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775776323.899237:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1775776323.899242:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776323.899247:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1775776323.899249:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776323.899249:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776323.899252:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1775776323.899254:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:2.0:1775776323.899255:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776323.899259:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776323.899260:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775776323.899263:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776323.899267:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776323.899270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776323.899273:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776323.899276:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13206 00000400:00000200:2.0:1775776323.899289:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776323.899297:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776323.899300:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776323.899303:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776323.899307:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776323.899309:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776323.899312:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776323.899317:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776323.899319:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776323.899321:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776323.899324:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776323.899326:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776323.899327:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776323.899329:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776323.899331:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776323.899332:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776323.899333:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776323.899335:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776323.899337:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776323.899338:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776323.899342:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776323.899344:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776323.899346:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776323.899348:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776323.899349:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776323.899364:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775776323.899367:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776323.899371:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776323.899372:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776323.899375:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776323.899377:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776323.899382:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776323.899391:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775776323.899399:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776323.899410:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710405760/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776323.899421:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776323.899428:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776323.899433:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710405760/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776323.899454:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776323.899457:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776323.899459:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776323.899460:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776323.899461:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776323.899463:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776323.899465:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13206 00000400:00000200:2.0:1775776323.899468:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776323.899470:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776323.899471:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776323.899473:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776323.899475:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776323.899476:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776323.899478:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776323.899480:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776323.899481:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776323.899482:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776323.899484:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776323.899485:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776323.899486:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776323.899488:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776323.899489:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776323.899491:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1775776323.899491:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776323.899493:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776323.899496:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776323.899497:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776323.899498:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776323.899500:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776323.899513:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776323.899518:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1775776323.899524:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1775776323.899525:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344aa00 x1862022710405760/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775776323.899528:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776323.899533:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776323.899538:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775776323.899540:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000008:00000001:2.0:1775776323.899541:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776323.899544:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776323.899544:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776323.899549:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 02000000:00000001:1.0:1775776323.899551:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:2.0:1775776323.899552:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776323.899555:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776323.899556:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776323.899556:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776323.899560:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776323.899562:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776323.899562:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776323.899564:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776323.899568:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710405760:192.168.201.115@tcp:250:kworker.0 02000000:00000010:2.0:1775776323.899569:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775776323.899574:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776323.899575:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776323.899579:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:2.0:1775776323.899579:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710405888/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1775776323.899583:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776323.899587:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1775776323.899590:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776323.899592:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776323.899594:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 00000100:00000040:2.0:1775776323.899595:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710405888/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775776323.899598:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776323.899605:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776323.899607:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00080000:2.0:1775776323.899608:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776323.899611:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000010:1.0:1775776323.899612:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000001:2.0:1775776323.899613:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776323.899615:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776323.899618:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000400:00000010:1.0:1775776323.899619:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00080000:2.0:1775776323.899620:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13206 00000400:00000200:2.0:1775776323.899624:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776323.899626:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000200:1.0:1775776323.899627:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710405760, portal 25 00000100:00080000:2.0:1775776323.899628:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776323.899631:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000040:2.0:1775776323.899632:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776323.899633:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662545920 : -123823047005696 : ffff8f623344aa00) 00000100:00000001:2.0:1775776323.899635:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776323.899638:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000040:1.0:1775776323.899641:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344aa00 x1862022710405760/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776348 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776323.899641:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776323.899645:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776323.899647:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776323.899651:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776323.899654:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000100:00000001:1.0:1775776323.899655:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1775776323.899655:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776323.899659:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:1.0:1775776323.899660:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776323.899663:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000040:2.0:1775776323.899664:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000200:1.0:1775776323.899666:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710405760, offset 0 00000100:00000001:2.0:1775776323.899667:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776323.899670:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776323.899672:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776323.899674:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776323.899675:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00080000:2.0:1775776323.899678:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776323.899680:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776323.899681:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776323.899683:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775776323.899686:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:2.0:1775776323.899686:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776323.899689:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776323.899691:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776323.899692:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776323.899694:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776323.899695:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000010:2.0:1775776323.899698:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000400:00000200:1.0:1775776323.899701:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:2.0:1775776323.899702:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776323.899704:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776323.899705:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776323.899706:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:2.0:1775776323.899707:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776323.899710:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776323.899711:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776323.899713:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 02000000:00000010:2.0:1775776323.899714:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000400:00000200:1.0:1775776323.899717:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:2.0:1775776323.899719:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776323.899723:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453722:453722:256:4294967295] 192.168.201.115@tcp LPNI seq info [453647:453647:8:4294967295] 00000100:00100000:2.0:1775776323.899725:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710406016/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776323.899730:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:2.0:1775776323.899736:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776323.899737:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776323.899742:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710406016/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776323.899747:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:2.0:1775776323.899756:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776323.899757:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000040:2.0:1775776323.899759:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776323.899762:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000010:1.0:1775776323.899763:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000100:00000040:2.0:1775776323.899764:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776323.899767:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.0:1775776323.899772:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775776323.899772:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13216) 00000800:00000200:1.0:1775776323.899777:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775776323.899782:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776323.899784:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775776323.899787:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776323.899789:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776323.899795:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776323.899799:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776323.899802:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776323.899804:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776323.899810:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776323.899813:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344aa00 x1862022710405760/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776348 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.899819:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776323.899821:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344aa00 x1862022710405760/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776348 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.899826:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776323.899827:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776323.899832:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776323.899835:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776323.899837:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307882758 00000100:00000010:1.0:1775776323.899840:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2600. 00000100:00000001:1.0:1775776323.899847:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776323.899852:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307882758 00000100:00000010:1.0:1775776323.899854:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2720. 00000100:00000001:1.0:1775776323.899856:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776323.899859:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776323.899861:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776323.899863:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776323.899865:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776323/real 1775776323] req@ffff8f623344aa00 x1862022710405760/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776323.899871:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776323.899872:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776323.899874:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776323.899875:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776323.899876:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776323.899877:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776323.899880:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344aa00 x1862022710405760/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776323.899884:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344aa00 x1862022710405760/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.899888:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776323.899889:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776323.899891:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776323.899893:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776323.899895:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344aa00 x1862022710405760/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.899899:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776323.899901:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344aa00 x1862022710405760/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.899905:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776323.899907:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776323.899908:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776323.899908:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776323.899910:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776323.899911:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776323.899913:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776323.899917:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776323.899918:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776323.899921:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344aa00 x1862022710405760/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776323.899925:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710405760:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776323.899928:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776323.899929:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776323.899931:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344aa00 x1862022710405760/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.899935:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776323.899937:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776323.899939:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776323.899940:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776323.899942:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776323.899942:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776323.899944:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776323.899945:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4cc00. 02000000:00000001:1.0:1775776323.899948:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776323.899949:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776323.899950:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344aa00. 00000100:00000001:1.0:1775776323.899953:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776323.899954:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776323.899955:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776323.899962:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776323.899963:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776323.899964:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776323.899965:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776323.899966:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776323.899968:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449500 x1862022710405888/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.899972:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776323.899973:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776323.899975:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776323.899976:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776323.899977:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776323.899979:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710405888:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776323.899981:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776323.899982:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776323.899983:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776323.899984:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776323.899985:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776323.899986:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776323.899988:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776323.899990:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776323.899992:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776323.899994:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710405888, portal 10 00000100:00000001:1.0:1775776323.899995:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776323.899996:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662540544 : -123823047011072 : ffff8f6233449500) 00000100:00000040:1.0:1775776323.900000:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449500 x1862022710405888/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776348 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.900004:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776323.900005:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776323.900007:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776323.900008:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710405888, offset 0 00000400:00000200:1.0:1775776323.900010:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776323.900012:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776323.900014:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776323.900015:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776323.900017:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776323.900019:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776323.900020:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776323.900022:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453723:453723:256:4294967295] 192.168.201.115@tcp LPNI seq info [453648:453648:8:4294967295] 00000400:00000200:1.0:1775776323.900025:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776323.900029:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776323.900032:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776323.900034:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776323.900036:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776323.900038:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775776323.900040:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776323.900041:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775776323.900043:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776323.900045:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776323.900047:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776323.900049:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776323.900050:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776323.900051:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776323.900053:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776323.900055:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449500 x1862022710405888/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776348 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.900060:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776323.900061:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449500 x1862022710405888/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776348 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.900065:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776323.900066:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776323.900067:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776323.900070:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776323.900071:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776323.900072:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307878923 00000100:00000001:1.0:1775776323.900074:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307882758 00000100:00000001:1.0:1775776323.900075:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307878923 00000100:00000001:1.0:1775776323.900077:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307882758 00000100:00000001:1.0:1775776323.900078:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776323.900079:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776323.900080:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776323.900082:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776323/real 1775776323] req@ffff8f6233449500 x1862022710405888/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776323.900087:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776323.900088:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776323.900089:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776323.900090:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776323.900091:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776323.900092:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776323.900094:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449500 x1862022710405888/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776323.900098:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449500 x1862022710405888/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.900101:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776323.900102:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776323.900103:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776323.900103:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776323.900105:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449500 x1862022710405888/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.900108:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776323.900110:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449500 x1862022710405888/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.900114:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776323.900115:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776323.900116:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776323.900116:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776323.900117:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776323.900121:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776323.900123:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776323.900124:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776323.900126:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776323.900128:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449500 x1862022710405888/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776323.900153:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710405888:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776323.900156:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776323.900158:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344ad80 x1862022710406016/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.900162:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776323.900163:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776323.900165:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776323.900166:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776323.900167:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776323.900168:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710406016:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776323.900170:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776323.900171:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776323.900172:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776323.900173:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776323.900175:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776323.900177:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776323.900179:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776323.900180:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776323.900181:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776323.900183:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710406016, portal 10 00000100:00000001:1.0:1775776323.900184:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776323.900185:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662546816 : -123823047004800 : ffff8f623344ad80) 00000100:00000040:1.0:1775776323.900189:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344ad80 x1862022710406016/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776348 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.900193:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776323.900194:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776323.900195:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776323.900197:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710406016, offset 0 00000400:00000200:1.0:1775776323.900200:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776323.900202:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776323.900203:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776323.900205:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776323.900207:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776323.900208:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776323.900210:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776323.900212:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453724:453724:256:4294967295] 192.168.201.115@tcp LPNI seq info [453649:453649:8:4294967295] 00000400:00000200:1.0:1775776323.900214:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776323.900218:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776323.900220:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776323.900222:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776323.900224:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776323.900226:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775776323.900228:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776323.900230:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775776323.900232:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776323.900234:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776323.900237:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776323.900240:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776323.900243:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776323.900244:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776323.900246:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776323.900249:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344ad80 x1862022710406016/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776348 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.900257:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776323.900265:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344ad80 x1862022710406016/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776348 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.900271:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776323.900273:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776323.900276:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776323.900279:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776323.900282:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776323.900284:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307878924 00000100:00000001:1.0:1775776323.900287:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307882758 00000100:00000001:1.0:1775776323.900289:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307878924 00000100:00000001:1.0:1775776323.900290:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307882758 00000100:00000001:1.0:1775776323.900292:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776323.900293:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776323.900294:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776323.900296:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776323/real 1775776323] req@ffff8f623344ad80 x1862022710406016/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776323.900301:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776323.900302:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776323.900303:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776323.900304:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776323.900305:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776323.900307:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776323.900308:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344ad80 x1862022710406016/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776323.900313:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344ad80 x1862022710406016/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.900317:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776323.900318:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776323.900319:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776323.900319:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776323.900321:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344ad80 x1862022710406016/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.900325:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776323.900326:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344ad80 x1862022710406016/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.900330:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776323.900332:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776323.900333:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776323.900334:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776323.900335:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776323.900336:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776323.900338:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776323.900340:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776323.900341:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776323.900343:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344ad80 x1862022710406016/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776323.900347:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710406016:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776323.900349:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776323.900350:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776323.900352:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449500 x1862022710405888/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.900355:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776323.900356:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776323.900357:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776323.900358:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776323.900359:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776323.900360:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776323.900361:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776323.900362:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4d400. 02000000:00000001:1.0:1775776323.900363:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776323.900364:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776323.900365:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449500. 00000100:00000001:1.0:1775776323.900367:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776323.900368:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776323.900369:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776323.900370:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344ad80 x1862022710406016/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776348 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776323.900384:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776323.900385:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776323.900386:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776323.900388:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776323.900388:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776323.900389:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776323.900390:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776323.900391:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4ec00. 02000000:00000001:1.0:1775776323.900392:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776323.900392:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776323.900393:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344ad80. 00000100:00000001:1.0:1775776323.900395:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776323.900396:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776323.900397:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776323.900399:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776323.900400:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776323.900401:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776323.900402:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776324.027261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776324.027267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776324.027274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776324.027275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776324.027279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776324.027281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776324.027339:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776324.027342:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776324.027345:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776324.923258:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776324.923267:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776324.923270:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776324.923277:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776324.923281:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776324.923286:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776324.923289:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776324.923292:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776324.923295:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776325.051244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776325.051251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776325.051258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776325.051260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776325.051264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776325.051265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776325.051318:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776325.051321:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776325.051324:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776325.947205:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776325.947214:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776325.947216:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776325.947222:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776325.947227:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776325.947230:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776325.947233:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776325.947236:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776325.947240:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776326.075232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776326.075238:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776326.075240:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776326.075242:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776326.075246:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776326.075247:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776326.075249:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776326.075250:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776326.075253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776326.075254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776326.075256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776326.075257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776326.075316:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776326.075325:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776326.075329:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776326.971231:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776326.971240:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776326.971242:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776326.971249:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776326.971253:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776326.971258:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776326.971260:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776326.971263:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776326.971265:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776327.099207:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776327.099213:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776327.099218:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776327.099219:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776327.099222:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776327.099223:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776327.099308:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776327.099312:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776327.099316:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776327.803294:0:214639:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307882758, at 4307882816 00000100:00000040:1.0:1775776327.803299:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307882758, at 4307882816 00000100:00000010:0.0:1775776327.803316:0:214639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2600. 00000100:00000010:1.0:1775776327.803324:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2720. 00000100:00000001:3.0:1775776327.995306:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776327.995316:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776327.995318:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776327.995326:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776327.995331:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776327.995336:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776327.995339:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776327.995343:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776327.995346:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776328.123283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776328.123289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776328.123294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776328.123295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776328.123299:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776328.123300:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776328.123384:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776328.123389:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776328.123393:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776329.019253:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776329.019262:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775776329.019263:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776329.019267:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776329.019267:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775776329.019271:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776329.019273:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775776329.019276:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776329.019281:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776329.019283:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776329.019286:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776329.019290:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775776329.019293:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13211 00000100:00000001:2.0:1775776329.019295:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776329.019300:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776329.019309:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776329.019318:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776329.019322:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776329.019326:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776329.019330:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776329.019333:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776329.019336:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776329.019340:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776329.019342:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776329.019345:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776329.019348:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776329.019351:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776329.019353:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776329.019356:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776329.019358:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776329.019360:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776329.019362:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776329.019365:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776329.019369:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776329.019371:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776329.019375:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776329.019379:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776329.019382:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776329.019394:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776329.019398:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776329.019411:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775776329.019431:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776329.019436:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776329.019438:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776329.019441:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776329.019445:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776329.019449:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776329.019468:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000001:3.0:1775776329.019477:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776329.019489:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710406144/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776329.019501:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776329.019507:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776329.019513:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710406144/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776329.019535:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776329.019538:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776329.019541:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776329.019543:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776329.019544:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776329.019547:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776329.019550:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13211 00000400:00000200:3.0:1775776329.019555:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776329.019558:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776329.019561:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776329.019564:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776329.019567:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776329.019569:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776329.019572:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776329.019574:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776329.019576:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776329.019578:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776329.019581:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776329.019584:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776329.019587:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776329.019591:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776329.019594:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776329.019597:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776329.019599:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:1.0:1775776329.019603:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1775776329.019604:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000001:1.0:1775776329.019612:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776329.019613:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776329.019616:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776329.019616:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776329.019620:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776329.019627:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1775776329.019630:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71c00 x1862022710406144/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775776329.019631:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776329.019635:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776329.019640:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776329.019651:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776329.019660:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776329.019661:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000010:3.0:1775776329.019665:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000100:00000001:1.0:1775776329.019666:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776329.019670:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776329.019673:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776329.019673:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776329.019674:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776329.019678:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776329.019679:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776329.019680:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776329.019681:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776329.019691:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776329.019692:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000001:3.0:1775776329.019699:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776329.019705:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710406144:192.168.201.115@tcp:250:kworker.0 00000100:00100000:3.0:1775776329.019706:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710406272/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776329.019728:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1775776329.019728:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776329.019734:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776329.019738:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1775776329.019738:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710406272/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775776329.019743:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776329.019753:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775776329.019766:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776329.019769:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 00000100:00080000:3.0:1775776329.019769:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1775776329.019777:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776329.019786:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00080000:3.0:1775776329.019787:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776329.019789:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:1.0:1775776329.019790:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00080000:3.0:1775776329.019792:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776329.019797:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000400:00000010:1.0:1775776329.019800:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00080000:3.0:1775776329.019801:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13211 00000400:00000200:3.0:1775776329.019808:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000200:1.0:1775776329.019811:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710406144, portal 25 00000400:00000200:3.0:1775776329.019812:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:1.0:1775776329.019820:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1775776329.019821:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776329.019825:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849011200 : -123823860540416 : ffff8f6202c71c00) 00000100:00000040:3.0:1775776329.019829:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776329.019833:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776329.019842:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000040:1.0:1775776329.019843:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71c00 x1862022710406144/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776329.019852:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776329.019861:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776329.019864:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776329.019867:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776329.019869:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776329.019870:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000001:1.0:1775776329.019881:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1775776329.019882:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:1.0:1775776329.019891:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776329.019894:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000010:1.0:1775776329.019896:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776329.019902:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710406144, offset 0 00000100:00000001:3.0:1775776329.019902:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776329.019909:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776329.019914:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776329.019921:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776329.019926:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776329.019932:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00080000:3.0:1775776329.019938:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000400:00000200:1.0:1775776329.019942:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:3.0:1775776329.019944:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776329.019949:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776329.019954:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1775776329.019955:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775776329.019960:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 02000000:00000001:3.0:1775776329.019962:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776329.019967:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1775776329.019967:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776329.019972:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1775776329.019973:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000008:00000001:3.0:1775776329.019983:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776329.019985:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453725:453725:256:4294967295] 192.168.201.115@tcp LPNI seq info [453650:453650:8:4294967295] 00000100:00000001:3.0:1775776329.019987:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776329.019994:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000010:3.0:1775776329.019994:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775776329.020001:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776329.020006:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776329.020012:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776329.020020:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776329.020024:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1775776329.020025:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 02000000:00000001:3.0:1775776329.020031:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776329.020036:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 02000000:00000010:3.0:1775776329.020037:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000800:00000010:1.0:1775776329.020041:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000100:00000001:3.0:1775776329.020043:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776329.020050:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710406400/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1775776329.020051:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776329.020062:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000100:00000001:3.0:1775776329.020075:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776329.020077:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776329.020082:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776329.020092:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710406400/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1775776329.020092:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775776329.020111:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776329.020120:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000100:00000001:3.0:1775776329.020131:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776329.020163:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776329.020168:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776329.020172:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776329.020176:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776329.020183:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13221) 00000400:00000200:1.0:1775776329.020189:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776329.020198:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776329.020205:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776329.020209:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776329.020216:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776329.020223:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71c00 x1862022710406144/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020234:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776329.020237:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71c00 x1862022710406144/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020242:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776329.020244:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776329.020249:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776329.020252:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776329.020255:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307887879 00000100:00000010:1.0:1775776329.020260:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2720. 00000100:00000001:1.0:1775776329.020265:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776329.020273:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307887879 00000100:00000010:1.0:1775776329.020275:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2a80. 00000100:00000001:1.0:1775776329.020277:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776329.020279:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776329.020282:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776329.020284:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776329.020286:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776329/real 1775776329] req@ffff8f6202c71c00 x1862022710406144/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776329.020299:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776329.020300:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776329.020302:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776329.020303:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776329.020304:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776329.020306:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776329.020309:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71c00 x1862022710406144/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776329.020314:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71c00 x1862022710406144/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020318:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776329.020319:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776329.020321:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776329.020323:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776329.020325:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71c00 x1862022710406144/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020330:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776329.020333:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71c00 x1862022710406144/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020338:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776329.020340:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776329.020341:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776329.020341:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776329.020343:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776329.020345:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776329.020347:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776329.020350:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776329.020351:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776329.020354:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71c00 x1862022710406144/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776329.020360:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710406144:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776329.020363:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776329.020365:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776329.020366:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71c00 x1862022710406144/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020370:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776329.020372:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776329.020374:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776329.020376:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776329.020378:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776329.020379:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776329.020388:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776329.020390:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37000. 02000000:00000001:1.0:1775776329.020393:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776329.020394:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776329.020396:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71c00. 00000100:00000001:1.0:1775776329.020398:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776329.020399:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776329.020400:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776329.020406:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776329.020408:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776329.020409:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776329.020411:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776329.020427:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776329.020431:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70e00 x1862022710406272/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020436:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776329.020438:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776329.020439:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776329.020441:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776329.020443:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776329.020445:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710406272:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776329.020448:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776329.020449:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776329.020451:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776329.020452:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776329.020454:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776329.020455:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776329.020458:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776329.020460:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776329.020463:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776329.020465:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710406272, portal 10 00000100:00000001:1.0:1775776329.020467:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776329.020468:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849007616 : -123823860544000 : ffff8f6202c70e00) 00000100:00000040:1.0:1775776329.020472:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70e00 x1862022710406272/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020476:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776329.020478:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776329.020479:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776329.020481:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710406272, offset 0 00000400:00000200:1.0:1775776329.020484:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776329.020486:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776329.020489:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776329.020491:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776329.020493:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776329.020495:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776329.020496:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776329.020498:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453726:453726:256:4294967295] 192.168.201.115@tcp LPNI seq info [453651:453651:8:4294967295] 00000400:00000200:1.0:1775776329.020501:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776329.020506:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776329.020508:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776329.020510:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776329.020513:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776329.020515:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775776329.020519:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776329.020520:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775776329.020523:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776329.020524:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776329.020526:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776329.020529:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776329.020531:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776329.020532:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776329.020533:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776329.020535:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70e00 x1862022710406272/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020541:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776329.020543:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70e00 x1862022710406272/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020547:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776329.020548:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776329.020550:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776329.020553:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776329.020555:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776329.020557:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307884044 00000100:00000001:1.0:1775776329.020559:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307887879 00000100:00000001:1.0:1775776329.020560:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307884044 00000100:00000001:1.0:1775776329.020562:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307887879 00000100:00000001:1.0:1775776329.020563:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776329.020565:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776329.020566:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776329.020568:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776329/real 1775776329] req@ffff8f6202c70e00 x1862022710406272/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776329.020573:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776329.020573:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776329.020575:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776329.020577:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776329.020578:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776329.020580:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776329.020582:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70e00 x1862022710406272/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776329.020587:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70e00 x1862022710406272/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020590:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776329.020591:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776329.020592:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776329.020593:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776329.020597:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70e00 x1862022710406272/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020603:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776329.020607:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70e00 x1862022710406272/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020614:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776329.020616:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776329.020618:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776329.020619:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776329.020621:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776329.020627:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776329.020630:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776329.020635:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776329.020637:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776329.020641:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70e00 x1862022710406272/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776329.020650:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710406272:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776329.020655:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776329.020659:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70000 x1862022710406400/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020667:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776329.020669:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776329.020671:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776329.020673:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776329.020675:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776329.020679:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710406400:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776329.020683:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776329.020685:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776329.020687:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776329.020689:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776329.020692:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776329.020695:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776329.020698:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776329.020701:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776329.020703:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776329.020706:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710406400, portal 10 00000100:00000001:1.0:1775776329.020707:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776329.020708:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004032 : -123823860547584 : ffff8f6202c70000) 00000100:00000040:1.0:1775776329.020713:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70000 x1862022710406400/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020718:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776329.020719:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776329.020721:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776329.020722:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710406400, offset 0 00000400:00000200:1.0:1775776329.020725:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776329.020727:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776329.020730:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776329.020732:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776329.020734:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776329.020736:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776329.020737:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776329.020740:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453727:453727:256:4294967295] 192.168.201.115@tcp LPNI seq info [453652:453652:8:4294967295] 00000400:00000200:1.0:1775776329.020742:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776329.020747:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776329.020749:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776329.020751:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776329.020754:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776329.020756:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775776329.020759:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776329.020760:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775776329.020763:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776329.020764:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776329.020767:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776329.020769:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776329.020771:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776329.020772:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776329.020774:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776329.020776:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70000 x1862022710406400/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020781:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776329.020783:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70000 x1862022710406400/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020788:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776329.020789:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776329.020791:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776329.020793:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776329.020795:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776329.020796:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307884044 00000100:00000001:1.0:1775776329.020798:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307887879 00000100:00000001:1.0:1775776329.020800:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307884044 00000100:00000001:1.0:1775776329.020801:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307887879 00000100:00000001:1.0:1775776329.020803:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776329.020804:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776329.020806:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776329.020808:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776329/real 1775776329] req@ffff8f6202c70000 x1862022710406400/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776329.020812:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776329.020813:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776329.020814:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776329.020816:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776329.020817:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776329.020818:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776329.020820:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70000 x1862022710406400/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776329.020825:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70000 x1862022710406400/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020828:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776329.020829:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776329.020830:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776329.020831:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776329.020833:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70000 x1862022710406400/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020837:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776329.020839:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70000 x1862022710406400/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020843:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776329.020845:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776329.020846:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776329.020847:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776329.020848:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776329.020850:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776329.020852:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776329.020854:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776329.020855:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776329.020858:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70000 x1862022710406400/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776329.020862:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710406400:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776329.020864:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776329.020865:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776329.020867:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70e00 x1862022710406272/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020871:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776329.020872:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776329.020873:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776329.020875:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776329.020876:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776329.020877:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776329.020878:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776329.020879:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34800. 02000000:00000001:1.0:1775776329.020881:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776329.020882:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776329.020883:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70e00. 00000100:00000001:1.0:1775776329.020885:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776329.020886:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776329.020887:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776329.020888:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70000 x1862022710406400/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776329.020892:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776329.020893:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776329.020894:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776329.020895:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776329.020896:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776329.020897:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776329.020898:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776329.020899:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34400. 02000000:00000001:1.0:1775776329.020901:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776329.020901:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776329.020902:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70000. 00000100:00000001:1.0:1775776329.020903:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776329.020904:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776329.020905:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776329.020907:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776329.020908:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776329.020909:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776329.020911:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776329.147276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776329.147283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776329.147288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776329.147289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776329.147293:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776329.147294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776329.147376:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776329.147381:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776329.147385:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776330.043282:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776330.043289:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776330.043291:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776330.043297:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776330.043301:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776330.043305:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776330.043307:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776330.043310:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776330.043312:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776330.171306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776330.171313:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776330.171322:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776330.171323:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776330.171326:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776330.171328:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776330.171367:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776330.171370:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776330.171374:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776331.067234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776331.067243:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776331.067245:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776331.067253:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776331.067259:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776331.067263:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776331.067267:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776331.067270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776331.067273:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776331.195249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776331.195254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776331.195258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776331.195261:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776331.195262:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776331.195265:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776331.195267:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776331.195269:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776331.195271:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776331.195272:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776331.195275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776331.195276:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776331.195277:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776331.195278:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776331.195279:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776331.195280:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776331.195281:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776331.195282:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776331.195394:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776331.195398:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776331.195402:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776332.091268:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776332.091276:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776332.091278:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776332.091285:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776332.091289:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776332.091293:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776332.091296:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776332.091299:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776332.091302:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776332.219258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776332.219263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776332.219269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776332.219270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776332.219274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776332.219276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776332.219346:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776332.219349:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776332.219352:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776332.923256:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307887879, at 4307887936 00000100:00000010:1.0:1775776332.923271:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2a80. 00000100:00000040:0.0:1775776332.923304:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307887879, at 4307887936 00000100:00000010:0.0:1775776332.923316:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2720. 00000100:00000001:2.0:1775776333.115256:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776333.115262:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776333.115264:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776333.115269:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776333.115271:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776333.115274:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776333.115277:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776333.115278:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776333.115281:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776333.243276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776333.243285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776333.243292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776333.243293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776333.243299:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776333.243300:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776333.243404:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776333.243409:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776333.243413:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776334.139247:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776334.139254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776334.139256:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776334.139261:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:3.0:1775776334.139262:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776334.139269:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776334.139274:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776334.139276:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776334.139277:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775776334.139284:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775776334.139285:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776334.139287:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776334.139292:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776334.139294:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776334.139301:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13216 00000400:00000200:3.0:1775776334.139312:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776334.139319:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776334.139335:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776334.139339:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776334.139343:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776334.139347:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776334.139349:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776334.139352:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776334.139354:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776334.139357:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776334.139360:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776334.139361:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776334.139363:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776334.139365:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776334.139367:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776334.139368:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776334.139370:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776334.139372:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776334.139375:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776334.139376:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776334.139379:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776334.139382:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776334.139384:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776334.139387:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776334.139389:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776334.139400:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73100. 00000020:00000040:3.0:1775776334.139403:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776334.139407:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776334.139409:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776334.139412:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776334.139415:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776334.139419:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776334.139427:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000100:00000001:3.0:1775776334.139433:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776334.139443:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73100 x1862022710406528/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776334.139457:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776334.139464:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776334.139469:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f67e6552] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73100 x1862022710406528/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776334.139487:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776334.139490:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776334.139493:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776334.139494:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776334.139495:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776334.139498:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776334.139500:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13216 00000400:00000200:3.0:1775776334.139503:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776334.139505:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776334.139507:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776334.139510:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776334.139512:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776334.139514:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776334.139517:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776334.139520:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776334.139521:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776334.139522:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776334.139524:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776334.139526:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776334.139530:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776334.139531:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776334.139539:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775776334.139539:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776334.139549:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776334.139549:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776334.139560:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776334.139561:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776334.139564:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000040:1.0:1775776334.139570:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73100 x1862022710406528/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775776334.139571:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776334.139581:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776334.139593:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776334.139601:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776334.139603:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776334.139603:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776334.139606:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776334.139607:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1775776334.139609:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776334.139611:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1775776334.139614:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775776334.139616:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775776334.139620:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776334.139622:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776334.139627:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000100:00100000:1.0:1775776334.139630:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710406528:192.168.201.115@tcp:250:kworker.0 00000020:00000040:3.0:1775776334.139632:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776334.139643:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1775776334.139645:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776334.139645:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776334.139651:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776334.139652:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776334.139653:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776334.139661:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776334.139661:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776334.139676:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776334.139678:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000100:00000001:3.0:1775776334.139682:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776334.139685:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776334.139692:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776334.139692:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710406656/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776334.139704:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775776334.139708:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776334.139712:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776334.139715:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000040:3.0:1775776334.139719:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710406656/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1775776334.139730:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000001:3.0:1775776334.139732:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776334.139737:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776334.139739:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000200:1.0:1775776334.139741:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710406528, portal 25 00000100:00000001:3.0:1775776334.139741:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776334.139745:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776334.139747:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776334.139748:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776334.139752:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849016576 : -123823860535040 : ffff8f6202c73100) 00000100:00080000:3.0:1775776334.139757:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13216 00000100:00000040:1.0:1775776334.139764:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73100 x1862022710406528/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775776334.139765:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776334.139776:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776334.139787:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776334.139801:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776334.139802:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775776334.139807:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776334.139814:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776334.139814:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776334.139822:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000010:1.0:1775776334.139825:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000020:00000001:3.0:1775776334.139825:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776334.139826:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776334.139831:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000200:1.0:1775776334.139834:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710406528, offset 0 00000020:00000040:3.0:1775776334.139834:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776334.139837:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776334.139845:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776334.139847:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776334.139849:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776334.139852:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776334.139855:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776334.139861:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776334.139862:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00080000:3.0:1775776334.139866:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776334.139870:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776334.139872:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776334.139875:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776334.139879:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776334.139880:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776334.139882:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776334.139885:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776334.139888:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776334.139889:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776334.139893:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000400:00000200:1.0:1775776334.139894:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000020:00000040:3.0:1775776334.139898:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:1.0:1775776334.139901:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1775776334.139907:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776334.139908:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776334.139913:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:3.0:1775776334.139914:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776334.139918:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776334.139920:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776334.139921:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 02000000:00000010:3.0:1775776334.139929:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000400:00000200:1.0:1775776334.139930:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:3.0:1775776334.139941:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776334.139944:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453728:453728:256:4294967295] 192.168.201.115@tcp LPNI seq info [453653:453653:8:4294967295] 00000400:00000200:1.0:1775776334.139954:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00100000:3.0:1775776334.139956:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710406784/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776334.139970:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776334.139985:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776334.139992:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000010:1.0:1775776334.139993:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000100:00000001:3.0:1775776334.139999:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776334.140007:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776334.140008:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710406784/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1775776334.140018:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000100:00000001:3.0:1775776334.140020:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776334.140030:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776334.140033:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776334.140036:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776334.140038:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776334.140044:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13226) 00000800:00000200:1.1:1775776334.140053:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776334.140058:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1775776334.140065:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776334.140067:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776334.140073:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776334.140087:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776334.140094:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776334.140096:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776334.140102:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776334.140107:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73100 x1862022710406528/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140119:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776334.140123:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73100 x1862022710406528/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140131:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776334.140159:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776334.140167:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776334.140172:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776334.140175:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307892999 00000100:00000010:1.0:1775776334.140181:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2a80. 00000100:00000001:1.0:1775776334.140188:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776334.140196:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307892999 00000100:00000010:1.0:1775776334.140199:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2f60. 00000100:00000001:1.0:1775776334.140202:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776334.140205:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776334.140207:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776334.140209:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776334.140213:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776334/real 1775776334] req@ffff8f6202c73100 x1862022710406528/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776334.140220:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776334.140222:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776334.140224:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776334.140226:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776334.140227:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776334.140229:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776334.140232:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73100 x1862022710406528/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776334.140238:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73100 x1862022710406528/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140242:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776334.140243:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776334.140245:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776334.140246:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776334.140248:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73100 x1862022710406528/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140253:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776334.140255:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73100 x1862022710406528/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140260:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776334.140263:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776334.140264:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776334.140265:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776334.140266:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776334.140268:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776334.140270:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776334.140273:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776334.140275:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776334.140277:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73100 x1862022710406528/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776334.140282:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710406528:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776334.140285:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776334.140287:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776334.140289:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73100 x1862022710406528/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140304:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776334.140307:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776334.140309:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776334.140312:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776334.140315:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776334.140317:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776334.140319:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776334.140322:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36000. 02000000:00000001:1.0:1775776334.140342:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776334.140344:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776334.140346:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73100. 00000100:00000001:1.0:1775776334.140350:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776334.140351:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776334.140353:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776334.140360:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776334.140362:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776334.140365:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776334.140366:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776334.140368:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776334.140378:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70000 x1862022710406656/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140384:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776334.140385:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776334.140387:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776334.140388:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776334.140390:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776334.140392:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710406656:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776334.140395:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776334.140396:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776334.140397:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776334.140399:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776334.140401:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776334.140403:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776334.140406:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776334.140408:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776334.140415:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776334.140418:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710406656, portal 10 00000100:00000001:1.0:1775776334.140419:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776334.140420:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004032 : -123823860547584 : ffff8f6202c70000) 00000100:00000040:1.0:1775776334.140425:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70000 x1862022710406656/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140429:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776334.140431:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776334.140432:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776334.140434:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710406656, offset 0 00000400:00000200:1.0:1775776334.140436:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776334.140439:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776334.140442:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776334.140444:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776334.140446:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776334.140447:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776334.140449:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776334.140451:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453729:453729:256:4294967295] 192.168.201.115@tcp LPNI seq info [453654:453654:8:4294967295] 00000400:00000200:1.0:1775776334.140454:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776334.140458:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776334.140461:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776334.140464:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776334.140466:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776334.140468:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775776334.140471:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776334.140473:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1775776334.140476:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776334.140477:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776334.140480:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776334.140482:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776334.140484:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776334.140485:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776334.140487:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776334.140489:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70000 x1862022710406656/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140495:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776334.140497:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70000 x1862022710406656/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140501:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776334.140502:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776334.140504:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776334.140507:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776334.140509:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776334.140511:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307889164 00000100:00000001:1.0:1775776334.140512:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307892999 00000100:00000001:1.0:1775776334.140514:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307889164 00000100:00000001:1.0:1775776334.140515:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307892999 00000100:00000001:1.0:1775776334.140517:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776334.140518:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776334.140519:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776334.140522:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776334/real 1775776334] req@ffff8f6202c70000 x1862022710406656/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776334.140526:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776334.140527:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776334.140528:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776334.140530:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776334.140531:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776334.140532:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776334.140535:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70000 x1862022710406656/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776334.140540:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70000 x1862022710406656/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140544:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776334.140545:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776334.140546:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776334.140547:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776334.140549:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70000 x1862022710406656/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140553:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776334.140555:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70000 x1862022710406656/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140559:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776334.140561:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776334.140562:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776334.140562:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776334.140564:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776334.140567:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776334.140569:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776334.140571:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776334.140572:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776334.140574:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70000 x1862022710406656/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776334.140579:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710406656:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776334.140582:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776334.140584:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70e00 x1862022710406784/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140593:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776334.140594:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776334.140596:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776334.140597:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776334.140598:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776334.140600:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710406784:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776334.140602:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776334.140603:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776334.140604:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776334.140605:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776334.140607:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776334.140608:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776334.140610:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776334.140612:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776334.140613:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776334.140615:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710406784, portal 10 00000100:00000001:1.0:1775776334.140616:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776334.140617:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849007616 : -123823860544000 : ffff8f6202c70e00) 00000100:00000040:1.0:1775776334.140621:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70e00 x1862022710406784/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140625:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776334.140627:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776334.140628:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776334.140630:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710406784, offset 0 00000400:00000200:1.0:1775776334.140632:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776334.140634:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776334.140636:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776334.140638:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776334.140639:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776334.140641:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776334.140642:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776334.140645:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453730:453730:256:4294967295] 192.168.201.115@tcp LPNI seq info [453655:453655:8:4294967295] 00000400:00000200:1.0:1775776334.140648:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776334.140652:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776334.140654:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776334.140656:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776334.140658:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776334.140660:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775776334.140663:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776334.140664:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1775776334.140667:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776334.140668:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776334.140670:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776334.140672:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776334.140674:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776334.140675:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776334.140676:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776334.140678:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70e00 x1862022710406784/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140683:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776334.140685:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70e00 x1862022710406784/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140689:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776334.140690:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776334.140692:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776334.140694:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776334.140695:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776334.140696:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307889164 00000100:00000001:1.0:1775776334.140698:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307892999 00000100:00000001:1.0:1775776334.140699:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307889164 00000100:00000001:1.0:1775776334.140701:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307892999 00000100:00000001:1.0:1775776334.140703:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776334.140704:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776334.140706:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776334.140708:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776334/real 1775776334] req@ffff8f6202c70e00 x1862022710406784/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776334.140713:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776334.140714:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776334.140715:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776334.140716:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776334.140717:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776334.140718:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776334.140720:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70e00 x1862022710406784/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776334.140725:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70e00 x1862022710406784/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140729:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776334.140730:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776334.140731:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776334.140732:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776334.140734:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70e00 x1862022710406784/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140738:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776334.140740:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70e00 x1862022710406784/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140744:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776334.140745:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776334.140746:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776334.140747:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776334.140748:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776334.140749:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776334.140750:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776334.140753:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776334.140754:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776334.140757:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70e00 x1862022710406784/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776334.140762:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710406784:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776334.140764:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776334.140765:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776334.140767:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70000 x1862022710406656/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140770:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776334.140771:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776334.140773:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776334.140774:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776334.140775:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776334.140776:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776334.140777:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776334.140778:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36400. 02000000:00000001:1.0:1775776334.140779:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776334.140780:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776334.140781:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70000. 00000100:00000001:1.0:1775776334.140783:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776334.140784:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776334.140785:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776334.140786:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70e00 x1862022710406784/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776334.140790:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776334.140791:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776334.140792:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776334.140793:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776334.140794:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776334.140794:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776334.140795:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776334.140797:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34c00. 02000000:00000001:1.0:1775776334.140798:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776334.140799:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776334.140800:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70e00. 00000100:00000001:1.0:1775776334.140802:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776334.140802:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776334.140803:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776334.140805:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776334.140806:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776334.140808:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776334.140808:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776334.267278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776334.267287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776334.267294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776334.267295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776334.267300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776334.267301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776334.267377:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776334.267383:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776334.267386:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776335.163264:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776335.163272:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776335.163274:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776335.163279:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776335.163282:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776335.163285:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776335.163288:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776335.163290:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776335.163292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776335.291243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776335.291250:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776335.291252:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776335.291256:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776335.291260:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776335.291263:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776335.291265:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776335.291268:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776335.291272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776335.291273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776335.291278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776335.291280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776335.291351:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776335.291355:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776335.291357:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1775776335.995265:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:0.0:1775776335.995273:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:0.0:1775776335.995278:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1775776335.995464:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776335.995479:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1775776335.995480:0:2404:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1775776335.995483:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776335.995488:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776336.187227:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776336.187232:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776336.187234:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776336.187239:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776336.187242:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776336.187245:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776336.187247:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776336.187249:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776336.187251:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776336.315221:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776336.315226:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776336.315230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776336.315231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776336.315233:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776336.315234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776336.315311:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776336.315315:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776336.315318:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776337.211246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776337.211251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776337.211253:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776337.211258:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776337.211261:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776337.211263:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776337.211265:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776337.211267:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776337.211269:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776337.339202:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776337.339206:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776337.339211:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776337.339213:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776337.339217:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776337.339218:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776337.339286:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776337.339289:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776337.339292:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776338.043238:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307892999, at 4307893056 00000100:00000040:0.0:1775776338.043251:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307892999, at 4307893056 00000100:00000010:1.0:1775776338.043251:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2f60. 00000100:00000010:0.0:1775776338.043282:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2a80. 00000100:00000001:1.0:1775776338.235250:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776338.235272:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776338.235274:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776338.235281:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776338.235284:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776338.235287:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776338.235290:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776338.235292:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776338.235295:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776338.363241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776338.363246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776338.363262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776338.363263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776338.363268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776338.363269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776338.363346:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776338.363350:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776338.363353:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776339.259249:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776339.259259:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776339.259263:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776339.259265:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775776339.259268:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776339.259271:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776339.259275:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776339.259284:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776339.259284:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13221 00000100:00100000:1.0:1775776339.259291:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000400:00000200:3.0:1775776339.259297:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776339.259301:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776339.259306:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1775776339.259307:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:1.0:1775776339.259312:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775776339.259313:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776339.259317:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776339.259319:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776339.259321:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776339.259324:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776339.259328:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776339.259331:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776339.259335:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776339.259338:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776339.259340:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776339.259343:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776339.259347:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776339.259350:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776339.259354:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776339.259356:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776339.259359:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776339.259361:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776339.259364:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776339.259367:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776339.259370:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776339.259373:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776339.259377:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776339.259379:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776339.259381:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776339.259382:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776339.259390:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775776339.259393:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776339.259398:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776339.259399:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776339.259403:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776339.259405:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776339.259410:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776339.259418:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775776339.259425:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776339.259435:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710406912/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776339.259446:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776339.259451:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776339.259456:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710406912/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776339.259478:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776339.259481:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776339.259484:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776339.259485:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776339.259486:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776339.259489:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776339.259491:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13221 00000400:00000200:3.0:1775776339.259494:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776339.259497:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776339.259499:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776339.259501:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776339.259504:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776339.259506:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776339.259507:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776339.259509:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776339.259510:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776339.259511:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776339.259512:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:1.0:1775776339.259514:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1775776339.259514:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776339.259518:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776339.259520:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775776339.259522:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776339.259524:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776339.259525:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776339.259528:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776339.259530:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776339.259532:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000040:1.0:1775776339.259535:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71c00 x1862022710406912/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775776339.259536:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776339.259540:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776339.259543:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776339.259549:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776339.259554:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776339.259555:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776339.259556:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776339.259558:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775776339.259561:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775776339.259563:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:3.0:1775776339.259564:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776339.259566:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776339.259567:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1775776339.259570:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 02000000:00000001:1.0:1775776339.259572:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776339.259574:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776339.259579:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1775776339.259580:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710406912:192.168.201.115@tcp:250:kworker.0 02000000:00000001:3.0:1775776339.259582:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776339.259587:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775776339.259589:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776339.259590:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776339.259594:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776339.259596:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776339.259600:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776339.259600:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 02000000:00000001:1.0:1775776339.259604:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775776339.259606:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776339.259612:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 00000100:00100000:3.0:1775776339.259614:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710407040/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1775776339.259616:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776339.259628:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776339.259629:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776339.259632:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776339.259635:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000040:3.0:1775776339.259636:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710407040/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1775776339.259645:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000001:3.0:1775776339.259654:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776339.259656:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710406912, portal 25 00000100:00080000:3.0:1775776339.259656:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776339.259662:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776339.259663:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776339.259664:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776339.259667:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849011200 : -123823860540416 : ffff8f6202c71c00) 00000100:00080000:3.0:1775776339.259675:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1775776339.259676:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71c00 x1862022710406912/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776339.259678:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776339.259683:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13221 00000400:00000200:3.0:1775776339.259691:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776339.259696:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:1.0:1775776339.259697:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1775776339.259699:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776339.259704:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776339.259705:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776339.259709:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1775776339.259711:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000040:3.0:1775776339.259711:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776339.259716:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000200:1.0:1775776339.259718:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710406912, offset 0 00000020:00000001:3.0:1775776339.259718:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776339.259720:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776339.259726:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776339.259729:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000400:00000200:1.0:1775776339.259730:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000020:00000001:3.0:1775776339.259731:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776339.259735:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776339.259738:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776339.259740:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776339.259743:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775776339.259744:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776339.259747:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776339.259750:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776339.259752:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000400:00000200:1.0:1775776339.259755:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:3.0:1775776339.259755:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776339.259757:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776339.259761:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775776339.259764:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1775776339.259764:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776339.259768:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776339.259770:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1775776339.259772:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000008:00000001:3.0:1775776339.259773:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776339.259776:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776339.259779:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000010:3.0:1775776339.259780:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775776339.259784:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:1.0:1775776339.259786:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 02000000:00000001:3.0:1775776339.259788:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776339.259790:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776339.259791:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776339.259794:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1775776339.259796:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453731:453731:256:4294967295] 192.168.201.115@tcp LPNI seq info [453656:453656:8:4294967295] 02000000:00000001:3.0:1775776339.259797:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776339.259801:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000100:00000001:3.0:1775776339.259805:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776339.259806:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00100000:3.0:1775776339.259811:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710407168/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776339.259822:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776339.259825:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776339.259835:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000040:3.0:1775776339.259835:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710407168/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1775776339.259851:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776339.259854:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776339.259858:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:1.0:1775776339.259859:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000100:00000040:3.0:1775776339.259867:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776339.259869:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.0:1775776339.259872:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776339.259873:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.0:1775776339.259879:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000100:00000040:3.0:1775776339.259880:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13231) 00000800:00000200:1.1:1775776339.259889:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776339.259893:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1775776339.259899:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776339.259901:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776339.259909:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776339.259914:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776339.259920:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776339.259922:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776339.259928:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776339.259933:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71c00 x1862022710406912/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.259943:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776339.259948:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71c00 x1862022710406912/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776364 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.259956:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776339.259958:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776339.259963:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776339.259968:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776339.259971:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307898118 00000100:00000010:1.0:1775776339.259976:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2f60. 00000100:00000001:1.0:1775776339.259983:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776339.259990:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307898118 00000100:00000010:1.0:1775776339.259994:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2120. 00000100:00000001:1.0:1775776339.259997:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776339.260000:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776339.260003:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776339.260006:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776339.260011:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776339/real 1775776339] req@ffff8f6202c71c00 x1862022710406912/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776339.260021:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776339.260023:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776339.260026:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776339.260029:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776339.260031:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776339.260033:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776339.260037:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71c00 x1862022710406912/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776339.260046:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71c00 x1862022710406912/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.260053:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776339.260055:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776339.260057:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776339.260059:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776339.260063:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71c00 x1862022710406912/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.260073:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776339.260078:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71c00 x1862022710406912/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.260086:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776339.260090:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776339.260091:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776339.260093:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776339.260095:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776339.260097:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776339.260101:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776339.260105:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776339.260108:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776339.260113:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71c00 x1862022710406912/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776339.260123:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710406912:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776339.260128:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776339.260130:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776339.260155:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71c00 x1862022710406912/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.260164:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776339.260167:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776339.260170:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776339.260172:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776339.260174:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776339.260176:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776339.260178:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776339.260181:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34c00. 02000000:00000001:1.0:1775776339.260185:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776339.260187:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776339.260189:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71c00. 00000100:00000001:1.0:1775776339.260192:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776339.260194:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776339.260196:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776339.260203:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776339.260205:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776339.260207:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776339.260209:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776339.260210:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776339.260215:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73480 x1862022710407040/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.260222:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776339.260224:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776339.260226:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776339.260228:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776339.260231:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776339.260253:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710407040:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776339.260257:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776339.260260:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776339.260261:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776339.260263:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776339.260267:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776339.260269:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776339.260273:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776339.260276:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776339.260280:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776339.260283:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710407040, portal 10 00000100:00000001:1.0:1775776339.260285:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776339.260287:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849017472 : -123823860534144 : ffff8f6202c73480) 00000100:00000040:1.0:1775776339.260294:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73480 x1862022710407040/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.260302:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776339.260304:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776339.260307:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776339.260310:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710407040, offset 0 00000400:00000200:1.0:1775776339.260315:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776339.260319:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776339.260323:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776339.260326:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776339.260329:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776339.260332:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776339.260335:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776339.260338:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453732:453732:256:4294967295] 192.168.201.115@tcp LPNI seq info [453657:453657:8:4294967295] 00000400:00000200:1.0:1775776339.260342:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776339.260349:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776339.260354:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776339.260357:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776339.260361:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776339.260364:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775776339.260369:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776339.260372:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1775776339.260376:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776339.260378:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776339.260381:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776339.260385:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776339.260389:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776339.260391:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776339.260394:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776339.260398:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73480 x1862022710407040/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.260407:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776339.260410:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73480 x1862022710407040/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776364 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.260418:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776339.260419:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776339.260423:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776339.260428:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776339.260431:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776339.260434:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307894284 00000100:00000001:1.0:1775776339.260437:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307898118 00000100:00000001:1.0:1775776339.260440:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307894284 00000100:00000001:1.0:1775776339.260442:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307898118 00000100:00000001:1.0:1775776339.260444:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776339.260447:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776339.260449:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776339.260453:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776339/real 1775776339] req@ffff8f6202c73480 x1862022710407040/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776339.260462:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776339.260463:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776339.260466:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776339.260468:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776339.260470:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776339.260472:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776339.260476:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73480 x1862022710407040/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776339.260486:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73480 x1862022710407040/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.260493:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776339.260495:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776339.260497:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776339.260499:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776339.260503:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73480 x1862022710407040/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.260511:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776339.260515:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73480 x1862022710407040/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.260523:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776339.260526:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776339.260528:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776339.260529:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776339.260532:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776339.260537:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776339.260540:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776339.260544:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776339.260547:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776339.260552:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73480 x1862022710407040/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776339.260560:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710407040:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776339.260565:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776339.260568:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70380 x1862022710407168/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.260576:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776339.260578:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776339.260580:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776339.260582:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776339.260585:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776339.260588:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710407168:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776339.260592:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776339.260594:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776339.260596:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776339.260598:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776339.260601:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776339.260603:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776339.260607:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776339.260610:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776339.260614:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776339.260617:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710407168, portal 10 00000100:00000001:1.0:1775776339.260619:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776339.260620:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004928 : -123823860546688 : ffff8f6202c70380) 00000100:00000040:1.0:1775776339.260626:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70380 x1862022710407168/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.260633:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776339.260635:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776339.260638:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776339.260641:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710407168, offset 0 00000400:00000200:1.0:1775776339.260646:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776339.260650:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776339.260654:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776339.260657:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776339.260660:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776339.260663:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776339.260665:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776339.260668:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453733:453733:256:4294967295] 192.168.201.115@tcp LPNI seq info [453658:453658:8:4294967295] 00000400:00000200:1.0:1775776339.260673:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776339.260679:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776339.260684:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776339.260688:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776339.260692:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776339.260694:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775776339.260699:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776339.260702:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1775776339.260706:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776339.260708:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776339.260710:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776339.260714:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776339.260717:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776339.260719:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776339.260722:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776339.260725:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70380 x1862022710407168/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.260733:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776339.260736:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70380 x1862022710407168/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776364 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.260744:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776339.260745:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776339.260748:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776339.260751:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776339.260755:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776339.260757:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307894284 00000100:00000001:1.0:1775776339.260759:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307898118 00000100:00000001:1.0:1775776339.260761:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307894284 00000100:00000001:1.0:1775776339.260764:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307898118 00000100:00000001:1.0:1775776339.260766:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776339.260768:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776339.260770:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776339.260774:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776339/real 1775776339] req@ffff8f6202c70380 x1862022710407168/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776339.260783:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776339.260785:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776339.260787:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776339.260790:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776339.260792:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776339.260794:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776339.260797:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70380 x1862022710407168/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776339.260806:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70380 x1862022710407168/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.260814:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776339.260815:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776339.260817:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776339.260818:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776339.260822:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70380 x1862022710407168/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.260829:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776339.260832:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70380 x1862022710407168/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.260840:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776339.260842:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776339.260844:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776339.260845:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776339.260847:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776339.260849:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776339.260852:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776339.260856:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776339.260858:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776339.260862:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70380 x1862022710407168/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776339.260870:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710407168:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776339.260874:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776339.260876:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776339.260879:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73480 x1862022710407040/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.260885:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776339.260887:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776339.260889:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776339.260892:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776339.260893:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776339.260895:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776339.260897:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776339.260899:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36400. 02000000:00000001:1.0:1775776339.260902:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776339.260903:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776339.260905:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73480. 00000100:00000001:1.0:1775776339.260907:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776339.260908:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776339.260910:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776339.260913:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70380 x1862022710407168/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776364 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776339.260921:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776339.260922:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776339.260924:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776339.260927:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776339.260928:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776339.260929:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776339.260932:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776339.260933:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36000. 02000000:00000001:1.0:1775776339.260936:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776339.260937:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776339.260938:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70380. 00000100:00000001:1.0:1775776339.260940:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776339.260942:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776339.260943:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776339.260947:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776339.260948:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776339.260950:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776339.260952:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776339.387272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776339.387278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776339.387286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776339.387287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776339.387291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776339.387293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776339.387350:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776339.387353:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776339.387356:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776340.283279:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776340.283286:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776340.283288:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776340.283292:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776340.283295:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776340.283298:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776340.283300:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776340.283302:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776340.283304:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776340.411250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776340.411256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776340.411262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776340.411266:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776340.411269:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776340.411272:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776340.411275:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776340.411278:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776340.411280:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776340.411282:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776340.411287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776340.411288:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776340.411289:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776340.411291:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776340.411293:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776340.411294:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776340.411296:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776340.411298:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776340.411350:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776340.411353:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776340.411356:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776341.307239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776341.307245:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776341.307247:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776341.307251:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776341.307254:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776341.307257:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776341.307259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776341.307262:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776341.307264:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776341.435276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776341.435281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776341.435285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776341.435286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776341.435289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776341.435290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776341.435354:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776341.435359:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776341.435362:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776342.331280:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776342.331289:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776342.331291:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776342.331299:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776342.331304:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776342.331309:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776342.331312:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776342.331315:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776342.331319:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776342.459239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776342.459245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776342.459251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776342.459252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776342.459256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776342.459257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776342.459308:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776342.459312:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776342.459315:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776343.163238:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307898118, at 4307898176 00000100:00000010:1.0:1775776343.163258:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2120. 00000100:00000040:0.0:1775776343.163259:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307898118, at 4307898176 00000100:00000010:0.0:1775776343.163269:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2f60. 00000100:00000001:1.0:1775776343.355271:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776343.355280:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776343.355282:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776343.355291:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776343.355296:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776343.355301:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776343.355305:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776343.355308:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776343.355312:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776343.483287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776343.483296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776343.483306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776343.483307:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776343.483313:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776343.483314:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776343.483384:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776343.483390:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776343.483394:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776344.379292:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776344.379294:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776344.379300:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776344.379302:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776344.379306:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1775776344.379308:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776344.379310:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776344.379314:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776344.379316:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776344.379320:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776344.379322:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776344.379326:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776344.379330:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775776344.379333:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13226 00000100:00000001:1.0:1775776344.379335:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776344.379347:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776344.379356:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776344.379361:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776344.379366:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776344.379369:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776344.379372:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776344.379375:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776344.379380:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776344.379382:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776344.379385:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776344.379388:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776344.379391:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776344.379394:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776344.379397:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776344.379399:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776344.379401:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776344.379403:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776344.379407:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776344.379410:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776344.379413:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776344.379417:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776344.379422:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776344.379425:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776344.379429:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776344.379432:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776344.379444:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775776344.379448:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776344.379452:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776344.379455:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776344.379458:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776344.379461:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776344.379466:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776344.379477:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000001:3.0:1775776344.379485:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776344.379496:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710407296/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776344.379508:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776344.379514:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776344.379519:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710407296/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776344.379545:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776344.379548:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776344.379552:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776344.379554:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776344.379556:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776344.379559:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776344.379563:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13226 00000100:00000001:1.0:1775776344.379567:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1775776344.379568:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776344.379571:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1775776344.379574:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:1.0:1775776344.379575:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1775776344.379579:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776344.379583:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71500 x1862022710407296/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776344.379588:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776344.379594:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776344.379598:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1775776344.379599:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776344.379601:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776344.379605:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776344.379607:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1775776344.379610:0:191261:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000001:1.0:1775776344.379611:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1775776344.379613:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 02000000:00000001:1.0:1775776344.379614:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775776344.379617:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00100000:1.0:1775776344.379619:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710407296:192.168.201.115@tcp:250:kworker.0 00000020:00000040:3.0:1775776344.379621:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776344.379625:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776344.379627:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776344.379633:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1775776344.379634:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776344.379638:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775776344.379639:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1775776344.379640:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776344.379643:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776344.379644:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776344.379649:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:3.0:1775776344.379649:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776344.379654:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776344.379657:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776344.379659:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:3.0:1775776344.379659:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776344.379664:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776344.379668:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776344.379671:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776344.379673:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776344.379675:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000010:1.0:1775776344.379680:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000008:00100000:3.0:1775776344.379681:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776344.379686:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776344.379689:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776344.379694:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000020:00000040:3.0:1775776344.379696:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776344.379699:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000010:1.0:1775776344.379700:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 02000000:00000001:3.0:1775776344.379701:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776344.379703:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776344.379705:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776344.379706:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776344.379713:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000100:00000200:1.0:1775776344.379717:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710407296, portal 25 00000100:00000001:3.0:1775776344.379717:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776344.379722:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776344.379725:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849009408 : -123823860542208 : ffff8f6202c71500) 00000100:00100000:3.0:1775776344.379727:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710407424/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1775776344.379734:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71500 x1862022710407296/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776369 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776344.379742:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776344.379745:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775776344.379747:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776344.379751:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776344.379755:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000040:3.0:1775776344.379756:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710407424/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776344.379758:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710407296, offset 0 00000400:00000200:1.0:1775776344.379769:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776344.379775:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776344.379779:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776344.379781:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00080000:3.0:1775776344.379786:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776344.379788:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1775776344.379790:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00080000:3.0:1775776344.379791:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776344.379796:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775776344.379797:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:3.0:1775776344.379801:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13226 00000400:00000200:1.0:1775776344.379805:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:3.0:1775776344.379809:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776344.379810:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776344.379815:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:3.0:1775776344.379818:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000400:00000200:1.0:1775776344.379823:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453734:453734:256:4294967295] 192.168.201.115@tcp LPNI seq info [453659:453659:8:4294967295] 00000100:00080000:3.0:1775776344.379824:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776344.379831:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776344.379832:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776344.379836:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776344.379839:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776344.379841:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776344.379847:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000020:00000001:3.0:1775776344.379847:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776344.379850:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776344.379853:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:1.0:1775776344.379857:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000020:00000040:3.0:1775776344.379857:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776344.379862:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000010:1.0:1775776344.379863:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000020:00000001:3.0:1775776344.379866:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776344.379871:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000800:00000200:1.0:1775776344.379874:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775776344.379876:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776344.379880:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776344.379882:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000200:1.0:1775776344.379887:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000020:00000040:3.0:1775776344.379887:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776344.379891:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776344.379897:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776344.379898:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:3.0:1775776344.379900:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776344.379905:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.1:1775776344.379905:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 02000000:00000001:3.0:1775776344.379909:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776344.379912:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000100:1.0:1775776344.379916:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000008:00100000:3.0:1775776344.379916:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000010:1.0:1775776344.379920:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000008:00000001:3.0:1775776344.379921:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776344.379926:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776344.379930:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000400:00000200:1.0:1775776344.379931:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:3.0:1775776344.379933:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776344.379938:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776344.379941:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776344.379942:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1775776344.379945:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776344.379948:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776344.379950:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776344.379951:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776344.379955:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 02000000:00000010:3.0:1775776344.379955:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000001:3.0:1775776344.379962:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776344.379967:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:3.0:1775776344.379969:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710407552/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1775776344.379975:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71500 x1862022710407296/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776369 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776344.379985:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776344.379990:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776344.379993:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776344.379997:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710407552/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776344.379999:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71500 x1862022710407296/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776369 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776344.380025:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776344.380026:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776344.380030:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1775776344.380031:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776344.380037:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776344.380040:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776344.380042:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776344.380046:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1775776344.380051:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776344.380055:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307903238 00000100:00000040:3.0:1775776344.380057:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13237) 00000100:00000010:1.0:1775776344.380061:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2120. 00000100:00000001:1.0:1775776344.380070:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776344.380078:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307903238 00000100:00000010:1.0:1775776344.380080:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e25a0. 00000100:00000001:1.0:1775776344.380083:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776344.380086:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776344.380087:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776344.380090:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776344.380093:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776344/real 1775776344] req@ffff8f6202c71500 x1862022710407296/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776344.380101:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776344.380103:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776344.380105:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776344.380107:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776344.380108:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776344.380111:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776344.380113:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71500 x1862022710407296/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776344.380120:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71500 x1862022710407296/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776344.380124:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776344.380126:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776344.380128:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776344.380130:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776344.380132:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71500 x1862022710407296/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776344.380170:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776344.380174:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71500 x1862022710407296/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776344.380180:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776344.380182:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776344.380184:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776344.380185:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776344.380187:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776344.380189:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776344.380191:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776344.380194:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776344.380195:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776344.380198:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71500 x1862022710407296/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776344.380204:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710407296:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776344.380207:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776344.380209:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776344.380211:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71500 x1862022710407296/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776344.380215:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776344.380217:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776344.380219:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776344.380222:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776344.380223:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776344.380225:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776344.380227:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776344.380229:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34800. 02000000:00000001:1.0:1775776344.380233:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776344.380235:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776344.380236:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71500. 00000100:00000001:1.0:1775776344.380239:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776344.380240:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776344.380241:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776344.380248:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776344.380249:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776344.380251:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776344.380252:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776344.380253:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776344.380255:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70a80 x1862022710407424/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776344.380260:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776344.380261:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776344.380263:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776344.380264:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776344.380266:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776344.380267:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710407424:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776344.380269:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776344.380271:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776344.380272:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776344.380273:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776344.380275:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776344.380276:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776344.380279:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776344.380281:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776344.380283:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776344.380286:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710407424, portal 10 00000100:00000001:1.0:1775776344.380287:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776344.380288:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849006720 : -123823860544896 : ffff8f6202c70a80) 00000100:00000040:1.0:1775776344.380293:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70a80 x1862022710407424/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776369 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776344.380298:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776344.380300:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776344.380301:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776344.380303:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710407424, offset 0 00000400:00000200:1.0:1775776344.380306:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776344.380309:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776344.380313:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776344.380316:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776344.380318:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776344.380321:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776344.380323:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776344.380327:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453735:453735:256:4294967295] 192.168.201.115@tcp LPNI seq info [453660:453660:8:4294967295] 00000400:00000200:1.0:1775776344.380331:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776344.380338:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776344.380343:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776344.380347:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776344.380352:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776344.380355:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775776344.380360:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776344.380363:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1775776344.380367:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776344.380369:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776344.380372:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776344.380376:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776344.380379:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776344.380381:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776344.380385:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776344.380387:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70a80 x1862022710407424/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776369 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776344.380393:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776344.380395:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70a80 x1862022710407424/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776369 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776344.380404:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776344.380406:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776344.380408:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776344.380412:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776344.380415:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776344.380416:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307899404 00000100:00000001:1.0:1775776344.380418:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307903238 00000100:00000001:1.0:1775776344.380420:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307899404 00000100:00000001:1.0:1775776344.380422:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307903238 00000100:00000001:1.0:1775776344.380424:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776344.380425:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776344.380427:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776344.380429:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776344/real 1775776344] req@ffff8f6202c70a80 x1862022710407424/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776344.380434:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776344.380435:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776344.380436:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776344.380438:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776344.380439:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776344.380441:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776344.380443:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70a80 x1862022710407424/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776344.380449:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70a80 x1862022710407424/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776344.380452:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776344.380453:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776344.380454:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776344.380455:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776344.380457:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70a80 x1862022710407424/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776344.380462:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776344.380465:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70a80 x1862022710407424/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776344.380469:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776344.380471:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776344.380472:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776344.380473:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776344.380474:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776344.380479:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776344.380480:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776344.380482:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776344.380483:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776344.380485:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70a80 x1862022710407424/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776344.380490:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710407424:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776344.380493:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776344.380495:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70700 x1862022710407552/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776344.380500:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776344.380501:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776344.380502:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776344.380503:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776344.380504:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776344.380506:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710407552:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776344.380508:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776344.380509:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776344.380511:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776344.380512:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776344.380515:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776344.380516:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776344.380518:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776344.380520:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776344.380522:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776344.380524:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710407552, portal 10 00000100:00000001:1.0:1775776344.380526:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776344.380527:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849005824 : -123823860545792 : ffff8f6202c70700) 00000100:00000040:1.0:1775776344.380531:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70700 x1862022710407552/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776369 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776344.380535:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776344.380537:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776344.380538:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776344.380540:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710407552, offset 0 00000400:00000200:1.0:1775776344.380552:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776344.380557:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776344.380560:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776344.380562:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776344.380564:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776344.380566:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776344.380567:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776344.380570:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453736:453736:256:4294967295] 192.168.201.115@tcp LPNI seq info [453661:453661:8:4294967295] 00000400:00000200:1.0:1775776344.380573:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776344.380577:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776344.380585:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776344.380588:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776344.380595:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776344.380598:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775776344.380602:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776344.380605:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1775776344.380609:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776344.380611:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776344.380614:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776344.380618:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776344.380622:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776344.380624:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776344.380627:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776344.380631:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70700 x1862022710407552/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776369 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776344.380640:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776344.380643:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70700 x1862022710407552/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776369 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776344.380650:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776344.380652:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776344.380655:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776344.380658:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776344.380663:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776344.380665:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307899404 00000100:00000001:1.0:1775776344.380668:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307903238 00000100:00000001:1.0:1775776344.380670:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307899404 00000100:00000001:1.0:1775776344.380673:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307903238 00000100:00000001:1.0:1775776344.380675:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776344.380677:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776344.380679:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776344.380683:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776344/real 1775776344] req@ffff8f6202c70700 x1862022710407552/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776344.380693:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776344.380695:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776344.380697:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776344.380700:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776344.380701:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776344.380704:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776344.380706:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70700 x1862022710407552/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776344.380712:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70700 x1862022710407552/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776344.380717:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776344.380718:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776344.380719:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776344.380720:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776344.380722:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70700 x1862022710407552/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776344.380726:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776344.380728:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70700 x1862022710407552/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776344.380733:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776344.380735:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776344.380736:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776344.380736:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776344.380737:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776344.380739:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776344.380740:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776344.380742:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776344.380744:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776344.380746:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70700 x1862022710407552/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776344.380750:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710407552:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776344.380753:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776344.380755:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776344.380757:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70a80 x1862022710407424/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776344.380761:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776344.380763:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776344.380764:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776344.380766:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776344.380767:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776344.380768:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776344.380769:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776344.380771:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34400. 02000000:00000001:1.0:1775776344.380772:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776344.380774:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776344.380775:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70a80. 00000100:00000001:1.0:1775776344.380776:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776344.380777:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776344.380778:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776344.380780:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70700 x1862022710407552/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776369 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776344.380785:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776344.380785:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776344.380786:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776344.380788:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776344.380788:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776344.380789:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776344.380790:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776344.380791:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37000. 02000000:00000001:1.0:1775776344.380792:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776344.380793:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776344.380794:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70700. 00000100:00000001:1.0:1775776344.380795:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776344.380795:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776344.380797:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776344.380799:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776344.380800:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776344.380801:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776344.380803:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776344.507272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776344.507282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776344.507288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776344.507290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776344.507293:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776344.507294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776344.507397:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776344.507404:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776344.507409:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776345.403272:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776345.403279:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776345.403281:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776345.403286:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776345.403290:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776345.403294:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776345.403297:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776345.403299:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776345.403302:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776345.531292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776345.531312:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776345.531315:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776345.531319:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776345.531325:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776345.531328:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776345.531330:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776345.531333:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776345.531338:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776345.531340:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776345.531345:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776345.531346:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776345.531422:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776345.531427:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776345.531431:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776346.427252:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776346.427259:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776346.427261:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776346.427267:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776346.427272:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776346.427275:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776346.427292:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776346.427295:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776346.427298:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776346.555239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776346.555247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776346.555254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776346.555256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776346.555260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776346.555261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776346.555317:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776346.555321:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776346.555324:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776347.451263:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776347.451271:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776347.451274:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776347.451281:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776347.451286:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776347.451290:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776347.451301:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776347.451304:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776347.451307:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776347.579284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776347.579293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776347.579300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776347.579301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776347.579305:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776347.579306:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776347.579366:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776347.579372:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776347.579376:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776348.283262:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307903238, at 4307903296 00000100:00000010:1.0:1775776348.283279:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e25a0. 00000100:00000040:0.0:1775776348.283286:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307903238, at 4307903296 00000100:00000010:0.0:1775776348.283295:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2120. 00000100:00000001:1.0:1775776348.475243:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776348.475252:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776348.475255:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776348.475262:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776348.475267:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776348.475271:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776348.475274:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776348.475277:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776348.475280:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776348.603218:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776348.603227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776348.603235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776348.603237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776348.603241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776348.603242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776348.603299:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776348.603303:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776348.603305:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776349.499272:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776349.499274:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776349.499282:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1775776349.499285:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776349.499286:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776349.499291:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1775776349.499294:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:0.0:1775776349.499297:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776349.499300:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776349.499305:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776349.499311:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776349.499311:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776349.499318:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776349.499321:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776349.499324:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13232 00000400:00000200:0.0:1775776349.499338:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776349.499347:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776349.499352:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776349.499357:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776349.499360:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776349.499364:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776349.499366:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776349.499371:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776349.499373:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776349.499376:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776349.499380:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776349.499383:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776349.499385:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776349.499387:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776349.499389:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776349.499390:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776349.499391:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776349.499393:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776349.499396:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776349.499397:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776349.499400:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776349.499403:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776349.499405:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776349.499407:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776349.499409:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776349.499419:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062a00. 00000020:00000040:0.0:1775776349.499422:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776349.499426:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776349.499428:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776349.499430:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776349.499433:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776349.499439:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776349.499448:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4800. 00000100:00000001:0.0:1775776349.499456:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776349.499468:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062a00 x1862022710407680/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776349.499480:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776349.499486:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776349.499491:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009753f9f6] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062a00 x1862022710407680/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776349.499515:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776349.499518:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776349.499522:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776349.499524:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776349.499525:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776349.499528:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776349.499530:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13232 00000400:00000200:0.0:1775776349.499533:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776349.499541:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776349.499543:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776349.499548:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776349.499550:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776349.499553:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776349.499555:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776349.499556:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776349.499557:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776349.499559:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775776349.499559:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:0.0:1775776349.499562:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775776349.499564:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776349.499565:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1775776349.499566:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776349.499569:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776349.499570:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776349.499573:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776349.499574:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776349.499575:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776349.499577:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775776349.499583:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776349.499586:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776349.499587:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e062a00 x1862022710407680/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775776349.499588:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776349.499595:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776349.499604:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776349.499610:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776349.499612:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776349.499615:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1775776349.499617:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775776349.499621:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:0.0:1775776349.499622:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776349.499626:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776349.499626:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776349.499635:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776349.499636:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062d80. 00000020:00000040:0.0:1775776349.499642:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00100000:1.0:1775776349.499643:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e062a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710407680:192.168.201.115@tcp:250:kworker.0 02000000:00000001:0.0:1775776349.499650:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776349.499654:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776349.499655:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1775776349.499659:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776349.499661:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776349.499661:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776349.499668:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776349.499668:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776349.499678:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776349.499679:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6c00. 00000100:00000001:0.0:1775776349.499684:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776349.499688:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 00000100:00100000:0.0:1775776349.499695:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062d80 x1862022710407808/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1775776349.499695:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776349.499708:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775776349.499709:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776349.499713:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776349.499718:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009965f80b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062d80 x1862022710407808/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1775776349.499719:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000001:0.0:1775776349.499732:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776349.499733:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00080000:0.0:1775776349.499735:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776349.499741:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776349.499743:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776349.499744:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1775776349.499746:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710407680, portal 25 00000100:00000001:0.0:1775776349.499747:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776349.499752:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13232 00000100:00000001:1.0:1775776349.499753:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776349.499756:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574563840 : -123823134987776 : ffff8f622e062a00) 00000400:00000200:0.0:1775776349.499760:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776349.499768:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776349.499770:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776349.499776:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e062a00 x1862022710407680/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776374 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776349.499777:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776349.499783:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776349.499788:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776349.499794:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776349.499802:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776349.499805:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776349.499808:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776349.499811:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775776349.499814:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000001:1.0:1775776349.499815:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1775776349.499817:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775776349.499821:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776349.499829:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776349.499829:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000040:0.0:1775776349.499835:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000010:1.0:1775776349.499837:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000001:0.0:1775776349.499838:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776349.499843:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00000200:1.0:1775776349.499845:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710407680, offset 0 00000100:00080000:0.0:1775776349.499848:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776349.499853:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776349.499855:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776349.499857:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776349.499858:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776349.499863:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776349.499869:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1775776349.499869:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000008:00000001:0.0:1775776349.499877:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776349.499879:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776349.499882:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062680. 00000020:00000040:0.0:1775776349.499883:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776349.499886:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1775776349.499890:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:0.0:1775776349.499891:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776349.499898:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776349.499900:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776349.499902:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776349.499907:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5c00. 00000400:00000200:1.0:1775776349.499908:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:0.0:1775776349.499915:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776349.499917:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00100000:0.0:1775776349.499928:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062680 x1862022710407936/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776349.499928:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776349.499939:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1775776349.499949:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775776349.499950:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:0.0:1775776349.499957:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776349.499959:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453737:453737:256:4294967295] 192.168.201.115@tcp LPNI seq info [453662:453662:8:4294967295] 00000100:00000040:0.0:1775776349.499966:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000082c7985e] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062680 x1862022710407936/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776349.499977:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776349.499991:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776349.499994:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776349.500000:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776349.500007:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776349.500008:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000040:0.0:1775776349.500019:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776349.500030:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13242) 00000800:00000200:1.0:1775776349.500042:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776349.500050:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776349.500072:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776349.500076:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775776349.500083:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776349.500087:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775776349.500092:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776349.500095:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776349.500101:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776349.500106:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776349.500112:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776349.500114:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776349.500120:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776349.500125:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e062a00 x1862022710407680/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776374 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.500160:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776349.500165:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e062a00 x1862022710407680/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776374 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.500172:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776349.500174:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776349.500180:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776349.500185:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776349.500188:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307908359 00000100:00000010:1.0:1775776349.500194:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e25a0. 00000100:00000001:1.0:1775776349.500200:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776349.500207:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307908359 00000100:00000010:1.0:1775776349.500210:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2300. 00000100:00000001:1.0:1775776349.500214:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776349.500217:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776349.500220:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776349.500222:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776349.500227:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776349/real 1775776349] req@ffff8f622e062a00 x1862022710407680/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776349.500237:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776349.500238:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776349.500241:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776349.500244:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776349.500246:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776349.500248:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776349.500252:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e062a00 x1862022710407680/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776349.500262:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e062a00 x1862022710407680/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.500270:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776349.500271:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776349.500274:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776349.500276:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776349.500279:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e062a00 x1862022710407680/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.500286:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776349.500291:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e062a00 x1862022710407680/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.500298:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776349.500301:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776349.500303:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776349.500304:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776349.500307:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776349.500310:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776349.500313:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776349.500318:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776349.500320:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776349.500325:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e062a00 x1862022710407680/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776349.500333:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e062a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710407680:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776349.500337:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776349.500339:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776349.500342:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e062a00 x1862022710407680/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.500348:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776349.500351:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776349.500354:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776349.500358:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776349.500360:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776349.500362:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776349.500364:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776349.500366:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:1.0:1775776349.500370:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776349.500372:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776349.500374:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e062a00. 00000100:00000001:1.0:1775776349.500378:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776349.500379:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776349.500381:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776349.500388:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776349.500389:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776349.500392:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776349.500393:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776349.500395:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776349.500399:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e062d80 x1862022710407808/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.500406:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776349.500408:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776349.500410:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776349.500412:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776349.500414:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776349.500417:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e062d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710407808:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776349.500421:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776349.500423:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776349.500425:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776349.500427:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776349.500430:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776349.500431:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776349.500436:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776349.500439:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776349.500442:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776349.500445:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710407808, portal 10 00000100:00000001:1.0:1775776349.500448:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776349.500450:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574564736 : -123823134986880 : ffff8f622e062d80) 00000100:00000040:1.0:1775776349.500456:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e062d80 x1862022710407808/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776374 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.500462:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776349.500464:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776349.500466:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776349.500469:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710407808, offset 0 00000400:00000200:1.0:1775776349.500473:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776349.500477:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776349.500481:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776349.500485:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776349.500487:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776349.500490:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776349.500492:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776349.500495:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453738:453738:256:4294967295] 192.168.201.115@tcp LPNI seq info [453663:453663:8:4294967295] 00000400:00000200:1.0:1775776349.500500:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776349.500507:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776349.500512:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776349.500515:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776349.500519:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776349.500522:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775776349.500526:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776349.500529:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775776349.500534:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776349.500536:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776349.500539:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776349.500543:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776349.500546:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776349.500548:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776349.500551:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776349.500555:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e062d80 x1862022710407808/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776374 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.500570:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776349.500573:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e062d80 x1862022710407808/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776374 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.500580:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776349.500582:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776349.500585:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776349.500590:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776349.500593:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776349.500596:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307904524 00000100:00000001:1.0:1775776349.500599:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307908359 00000100:00000001:1.0:1775776349.500601:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307904524 00000100:00000001:1.0:1775776349.500604:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307908359 00000100:00000001:1.0:1775776349.500606:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776349.500608:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776349.500610:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776349.500615:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776349/real 1775776349] req@ffff8f622e062d80 x1862022710407808/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776349.500623:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776349.500625:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776349.500626:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776349.500629:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776349.500631:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776349.500633:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776349.500637:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e062d80 x1862022710407808/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776349.500645:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e062d80 x1862022710407808/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.500652:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776349.500654:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776349.500656:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776349.500657:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776349.500661:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e062d80 x1862022710407808/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.500667:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776349.500672:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e062d80 x1862022710407808/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.500679:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776349.500681:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776349.500683:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776349.500684:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776349.500686:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776349.500691:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776349.500694:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776349.500698:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776349.500701:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776349.500705:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e062d80 x1862022710407808/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776349.500714:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e062d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710407808:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776349.500719:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776349.500723:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e062680 x1862022710407936/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.500730:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776349.500731:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776349.500734:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776349.500736:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776349.500738:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776349.500741:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e062680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710407936:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776349.500745:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776349.500747:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776349.500749:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776349.500751:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776349.500755:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776349.500757:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776349.500760:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776349.500763:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776349.500766:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776349.500769:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710407936, portal 10 00000100:00000001:1.0:1775776349.500771:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776349.500773:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574562944 : -123823134988672 : ffff8f622e062680) 00000100:00000040:1.0:1775776349.500779:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e062680 x1862022710407936/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776374 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.500785:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776349.500788:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776349.500790:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776349.500793:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710407936, offset 0 00000400:00000200:1.0:1775776349.500797:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776349.500801:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776349.500805:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776349.500808:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776349.500811:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776349.500813:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776349.500816:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776349.500819:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453739:453739:256:4294967295] 192.168.201.115@tcp LPNI seq info [453664:453664:8:4294967295] 00000400:00000200:1.0:1775776349.500823:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776349.500829:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776349.500833:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776349.500837:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776349.500841:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776349.500844:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775776349.500848:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776349.500851:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775776349.500855:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776349.500857:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776349.500860:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776349.500863:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776349.500867:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776349.500868:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776349.500871:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776349.500874:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e062680 x1862022710407936/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776374 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.500882:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776349.500885:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e062680 x1862022710407936/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776374 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.500891:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776349.500892:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776349.500895:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776349.500898:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776349.500901:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776349.500903:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307904524 00000100:00000001:1.0:1775776349.500906:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307908359 00000100:00000001:1.0:1775776349.500908:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307904524 00000100:00000001:1.0:1775776349.500910:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307908359 00000100:00000001:1.0:1775776349.500912:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776349.500914:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776349.500916:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776349.500920:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776349/real 1775776349] req@ffff8f622e062680 x1862022710407936/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776349.500927:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776349.500929:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776349.500930:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776349.500932:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776349.500934:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776349.500936:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776349.500939:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e062680 x1862022710407936/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776349.500947:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e062680 x1862022710407936/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.500953:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776349.500954:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776349.500956:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776349.500957:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776349.500961:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e062680 x1862022710407936/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.500967:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776349.500970:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e062680 x1862022710407936/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.500977:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776349.500979:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776349.500980:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776349.500981:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776349.500983:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776349.500985:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776349.500987:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776349.500991:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776349.500993:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776349.500996:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e062680 x1862022710407936/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776349.501009:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e062680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710407936:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776349.501013:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776349.501015:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776349.501018:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e062d80 x1862022710407808/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.501024:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776349.501026:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776349.501028:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776349.501030:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776349.501032:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776349.501033:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776349.501035:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776349.501037:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e6c00. 02000000:00000001:1.0:1775776349.501039:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776349.501041:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776349.501042:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e062d80. 00000100:00000001:1.0:1775776349.501045:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776349.501047:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776349.501048:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776349.501052:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e062680 x1862022710407936/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776374 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776349.501076:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776349.501078:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776349.501080:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776349.501082:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776349.501084:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776349.501085:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776349.501087:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776349.501089:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e5c00. 02000000:00000001:1.0:1775776349.501091:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776349.501092:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776349.501094:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e062680. 00000100:00000001:1.0:1775776349.501096:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776349.501098:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776349.501100:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776349.501104:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776349.501105:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776349.501107:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776349.501109:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776349.627236:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776349.627242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776349.627248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776349.627249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776349.627252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776349.627253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776349.627318:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776349.627324:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776349.627327:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776350.523219:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776350.523229:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776350.523231:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776350.523238:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776350.523243:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776350.523248:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776350.523251:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776350.523254:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776350.523257:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776350.651286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776350.651293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776350.651300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776350.651304:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776350.651306:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776350.651309:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776350.651311:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776350.651314:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776350.651316:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776350.651318:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776350.651323:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776350.651325:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776350.651326:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776350.651327:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776350.651329:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776350.651331:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776350.651332:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776350.651334:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776350.651397:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776350.651402:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776350.651406:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776351.547275:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776351.547283:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776351.547284:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776351.547289:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776351.547292:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776351.547297:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776351.547299:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776351.547301:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776351.547303:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776351.675251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776351.675259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776351.675266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776351.675267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776351.675271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776351.675272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776351.675329:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776351.675332:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776351.675336:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776352.571296:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776352.571305:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776352.571307:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776352.571314:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776352.571319:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776352.571324:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776352.571327:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776352.571330:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776352.571333:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776352.699268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776352.699274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776352.699279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776352.699280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776352.699284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776352.699284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776352.699341:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776352.699346:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776352.699350:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776353.403251:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307908359, at 4307908416 00000100:00000010:1.0:1775776353.403268:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2300. 00000100:00000040:0.0:1775776353.403287:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307908359, at 4307908416 00000100:00000010:0.0:1775776353.403297:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e25a0. 00000100:00000001:1.0:1775776353.595320:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776353.595333:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776353.595335:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776353.595344:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776353.595348:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776353.595353:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776353.595356:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776353.595359:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776353.595362:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776353.723216:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776353.723222:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776353.723227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776353.723228:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776353.723232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776353.723233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776353.723286:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776353.723290:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776353.723293:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776354.619261:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776354.619262:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776354.619274:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776354.619277:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776354.619279:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776354.619282:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1775776354.619282:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:0.0:1775776354.619287:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776354.619287:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776354.619290:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776354.619292:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776354.619293:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776354.619296:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776354.619299:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776354.619301:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13237 00000400:00000200:0.0:1775776354.619312:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776354.619318:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776354.619321:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776354.619324:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776354.619327:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776354.619329:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776354.619331:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776354.619334:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776354.619336:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776354.619338:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776354.619341:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776354.619343:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776354.619345:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776354.619348:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776354.619349:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776354.619350:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776354.619351:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776354.619354:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776354.619356:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776354.619357:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776354.619360:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776354.619363:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776354.619365:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776354.619367:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776354.619368:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776354.619378:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060380. 00000020:00000040:0.0:1775776354.619380:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776354.619384:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776354.619385:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776354.619387:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776354.619390:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776354.619394:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776354.619402:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7000. 00000100:00000001:0.0:1775776354.619409:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776354.619418:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060380 x1862022710408064/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776354.619427:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776354.619432:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776354.619435:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000614ace82] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060380 x1862022710408064/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776354.619455:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776354.619457:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776354.619459:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776354.619461:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776354.619462:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776354.619464:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776354.619466:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13237 00000400:00000200:0.0:1775776354.619468:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776354.619470:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776354.619472:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776354.619474:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776354.619476:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776354.619477:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776354.619479:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776354.619480:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776354.619481:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776354.619482:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776354.619484:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775776354.619485:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775776354.619486:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775776354.619488:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776354.619489:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776354.619489:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1775776354.619491:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776354.619493:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776354.619494:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000001:1.0:1775776354.619494:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775776354.619498:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776354.619498:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1775776354.619499:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776354.619502:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776354.619507:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776354.619509:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776354.619510:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000040:1.0:1775776354.619510:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e060380 x1862022710408064/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775776354.619515:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776354.619520:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776354.619524:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776354.619526:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000010:0.0:1775776354.619527:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063800. 00000100:00000001:1.0:1775776354.619528:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775776354.619530:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776354.619532:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776354.619533:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776354.619534:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776354.619534:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776354.619536:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776354.619538:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776354.619538:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000010:0.0:1775776354.619540:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7c00. 02000000:00000001:1.0:1775776354.619543:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776354.619544:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776354.619552:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063800 x1862022710408192/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775776354.619552:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e060380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710408064:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776354.619564:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1775776354.619565:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776354.619567:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776354.619569:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:0.0:1775776354.619570:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000015d81d25] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063800 x1862022710408192/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775776354.619573:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776354.619578:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1775776354.619579:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776354.619582:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776354.619583:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776354.619584:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776354.619585:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000010:1.0:1775776354.619586:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 00000100:00000001:0.0:1775776354.619587:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776354.619591:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13237 02000000:00000001:1.0:1775776354.619591:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776354.619597:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776354.619598:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776354.619600:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1775776354.619602:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:0.0:1775776354.619603:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776354.619606:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776354.619608:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776354.619609:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000010:1.0:1775776354.619609:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000020:00000001:0.0:1775776354.619612:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776354.619614:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776354.619615:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776354.619616:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775776354.619618:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775776354.619619:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000010:1.0:1775776354.619620:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000040:0.0:1775776354.619621:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776354.619625:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776354.619626:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776354.619627:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776354.619628:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775776354.619629:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776354.619631:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776354.619631:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710408064, portal 25 02000000:00000001:0.0:1775776354.619634:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776354.619637:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776354.619637:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1775776354.619641:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776354.619641:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574553984 : -123823134997632 : ffff8f622e060380) 00000100:00000001:0.0:1775776354.619644:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776354.619646:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776354.619648:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776354.619649:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776354.619652:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e060380 x1862022710408064/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776379 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775776354.619654:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062680. 00000020:00000040:0.0:1775776354.619657:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776354.619661:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776354.619662:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776354.619665:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776354.619667:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776354.619669:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776354.619670:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000010:0.0:1775776354.619672:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5c00. 00000100:00000001:0.0:1775776354.619675:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776354.619676:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00100000:0.0:1775776354.619680:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062680 x1862022710408320/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1775776354.619681:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776354.619687:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710408064, offset 0 00000100:00000001:0.0:1775776354.619688:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776354.619690:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776354.619693:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000082c7985e] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062680 x1862022710408320/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776354.619700:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:0.0:1775776354.619701:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776354.619704:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776354.619707:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776354.619709:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776354.619713:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:0.0:1775776354.619714:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1775776354.619723:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000040:0.0:1775776354.619727:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13247) 00000400:00000200:1.0:1775776354.619730:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776354.619737:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776354.619742:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776354.619746:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776354.619750:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453740:453740:256:4294967295] 192.168.201.115@tcp LPNI seq info [453665:453665:8:4294967295] 00000400:00000200:1.0:1775776354.619754:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776354.619763:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776354.619768:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776354.619773:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776354.619780:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776354.619784:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775776354.619788:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776354.619791:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775776354.619794:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776354.619796:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776354.619802:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776354.619806:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776354.619811:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776354.619812:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776354.619818:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776354.619822:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e060380 x1862022710408064/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776379 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.619829:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776354.619831:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e060380 x1862022710408064/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776379 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.619836:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776354.619838:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776354.619843:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776354.619846:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776354.619848:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307913478 00000100:00000010:1.0:1775776354.619853:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2300. 00000100:00000001:1.0:1775776354.619858:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776354.619864:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307913478 00000100:00000010:1.0:1775776354.619866:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2de0. 00000100:00000001:1.0:1775776354.619868:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776354.619870:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776354.619872:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776354.619875:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776354.619877:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776354/real 1775776354] req@ffff8f622e060380 x1862022710408064/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776354.619883:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776354.619884:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776354.619886:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776354.619888:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776354.619889:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776354.619891:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776354.619893:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e060380 x1862022710408064/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776354.619899:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e060380 x1862022710408064/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.619902:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776354.619904:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776354.619905:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776354.619907:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776354.619909:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e060380 x1862022710408064/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.619913:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776354.619916:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e060380 x1862022710408064/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.619920:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776354.619922:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776354.619923:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776354.619924:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776354.619926:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776354.619927:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776354.619929:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776354.619932:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776354.619933:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776354.619935:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e060380 x1862022710408064/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776354.619940:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e060380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710408064:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776354.619945:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776354.619946:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776354.619948:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e060380 x1862022710408064/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.619952:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776354.619960:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776354.619962:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776354.619964:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776354.619965:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776354.619987:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776354.619990:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776354.619993:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e7000. 02000000:00000001:1.0:1775776354.619997:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776354.619999:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776354.620001:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e060380. 00000100:00000001:1.0:1775776354.620004:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776354.620005:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776354.620008:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776354.620016:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776354.620018:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776354.620021:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776354.620022:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776354.620024:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776354.620028:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e063800 x1862022710408192/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.620034:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776354.620036:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776354.620038:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776354.620039:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776354.620040:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776354.620042:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e063800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710408192:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776354.620045:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776354.620046:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776354.620047:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776354.620049:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776354.620051:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776354.620052:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776354.620055:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776354.620056:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776354.620059:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776354.620061:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710408192, portal 10 00000100:00000001:1.0:1775776354.620062:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776354.620063:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574567424 : -123823134984192 : ffff8f622e063800) 00000100:00000040:1.0:1775776354.620068:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e063800 x1862022710408192/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776379 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.620072:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776354.620073:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776354.620074:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776354.620076:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710408192, offset 0 00000400:00000200:1.0:1775776354.620078:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776354.620080:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776354.620082:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776354.620084:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776354.620086:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776354.620087:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776354.620089:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776354.620092:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453741:453741:256:4294967295] 192.168.201.115@tcp LPNI seq info [453666:453666:8:4294967295] 00000400:00000200:1.0:1775776354.620096:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776354.620102:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776354.620106:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776354.620109:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776354.620113:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776354.620115:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775776354.620119:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776354.620122:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775776354.620126:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776354.620127:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776354.620130:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776354.620151:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776354.620154:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776354.620155:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776354.620157:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776354.620159:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e063800 x1862022710408192/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776379 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.620164:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776354.620166:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e063800 x1862022710408192/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776379 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.620172:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776354.620173:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776354.620176:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776354.620180:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776354.620183:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776354.620184:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307909644 00000100:00000001:1.0:1775776354.620187:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307913478 00000100:00000001:1.0:1775776354.620189:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307909644 00000100:00000001:1.0:1775776354.620191:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307913478 00000100:00000001:1.0:1775776354.620193:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776354.620195:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776354.620197:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776354.620200:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776354/real 1775776354] req@ffff8f622e063800 x1862022710408192/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776354.620208:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776354.620210:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776354.620211:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776354.620213:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776354.620215:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776354.620217:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776354.620220:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e063800 x1862022710408192/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776354.620225:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e063800 x1862022710408192/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.620229:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776354.620230:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776354.620231:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776354.620232:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776354.620234:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e063800 x1862022710408192/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.620238:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776354.620240:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e063800 x1862022710408192/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.620244:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776354.620245:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776354.620246:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776354.620247:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776354.620248:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776354.620252:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776354.620260:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776354.620262:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776354.620263:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776354.620266:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e063800 x1862022710408192/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776354.620270:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e063800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710408192:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776354.620272:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776354.620274:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e062680 x1862022710408320/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.620283:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776354.620284:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776354.620285:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776354.620286:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776354.620287:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776354.620289:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e062680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710408320:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776354.620290:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776354.620291:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776354.620292:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776354.620293:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776354.620296:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776354.620297:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776354.620298:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776354.620300:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776354.620302:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776354.620304:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710408320, portal 10 00000100:00000001:1.0:1775776354.620305:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776354.620306:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574562944 : -123823134988672 : ffff8f622e062680) 00000100:00000040:1.0:1775776354.620309:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e062680 x1862022710408320/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776379 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.620313:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776354.620315:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776354.620316:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776354.620318:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710408320, offset 0 00000400:00000200:1.0:1775776354.620320:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776354.620322:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776354.620324:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776354.620326:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776354.620328:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776354.620329:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776354.620331:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776354.620333:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453742:453742:256:4294967295] 192.168.201.115@tcp LPNI seq info [453667:453667:8:4294967295] 00000400:00000200:1.0:1775776354.620336:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776354.620340:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776354.620343:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776354.620345:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776354.620347:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776354.620349:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775776354.620351:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776354.620353:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775776354.620355:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776354.620357:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776354.620358:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776354.620361:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776354.620362:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776354.620363:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776354.620365:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776354.620367:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e062680 x1862022710408320/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776379 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.620372:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776354.620373:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e062680 x1862022710408320/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776379 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.620378:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776354.620378:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776354.620380:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776354.620382:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776354.620384:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776354.620385:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307909644 00000100:00000001:1.0:1775776354.620386:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307913478 00000100:00000001:1.0:1775776354.620388:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307909644 00000100:00000001:1.0:1775776354.620389:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307913478 00000100:00000001:1.0:1775776354.620391:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776354.620393:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776354.620394:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776354.620396:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776354/real 1775776354] req@ffff8f622e062680 x1862022710408320/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776354.620401:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776354.620401:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776354.620402:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776354.620403:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776354.620404:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776354.620405:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776354.620407:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e062680 x1862022710408320/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776354.620412:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e062680 x1862022710408320/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.620415:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776354.620416:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776354.620417:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776354.620418:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776354.620419:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e062680 x1862022710408320/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.620423:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776354.620425:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e062680 x1862022710408320/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.620428:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776354.620429:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776354.620430:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776354.620431:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776354.620432:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776354.620433:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776354.620434:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776354.620436:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776354.620437:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776354.620440:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e062680 x1862022710408320/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776354.620444:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e062680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710408320:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776354.620446:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776354.620447:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776354.620448:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e063800 x1862022710408192/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.620452:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776354.620452:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776354.620453:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776354.620455:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776354.620456:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776354.620456:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776354.620458:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776354.620459:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e7c00. 02000000:00000001:1.0:1775776354.620460:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776354.620461:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776354.620462:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e063800. 00000100:00000001:1.0:1775776354.620464:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776354.620464:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776354.620465:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776354.620467:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e062680 x1862022710408320/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776379 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776354.620472:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776354.620474:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776354.620475:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776354.620477:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776354.620478:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776354.620479:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776354.620481:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776354.620483:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e5c00. 02000000:00000001:1.0:1775776354.620485:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776354.620486:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776354.620487:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e062680. 00000100:00000001:1.0:1775776354.620489:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776354.620490:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776354.620492:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776354.620495:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776354.620497:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776354.620499:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776354.620500:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776354.747269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776354.747275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776354.747280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776354.747281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776354.747284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776354.747285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776354.747351:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776354.747354:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776354.747357:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776355.643277:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776355.643286:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776355.643288:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776355.643294:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776355.643299:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776355.643303:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776355.643306:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776355.643309:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776355.643312:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776355.771278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776355.771287:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776355.771290:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776355.771293:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776355.771298:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776355.771300:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776355.771302:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776355.771304:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776355.771307:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776355.771308:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776355.771311:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776355.771312:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776355.771368:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776355.771371:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776355.771374:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776356.667258:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776356.667267:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776356.667269:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776356.667275:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776356.667278:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776356.667282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776356.667285:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776356.667287:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776356.667290:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776356.795263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776356.795270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776356.795276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776356.795278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776356.795281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776356.795282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776356.795382:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776356.795388:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776356.795393:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776357.691246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776357.691251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776357.691252:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776357.691257:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776357.691260:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776357.691263:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776357.691265:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776357.691267:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776357.691268:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776357.819272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776357.819278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776357.819282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776357.819283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776357.819286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776357.819287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776357.819365:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776357.819370:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776357.819373:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776358.523267:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307913478, at 4307913536 00000100:00000010:1.0:1775776358.523283:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2de0. 00000100:00000040:0.0:1775776358.523284:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307913478, at 4307913536 00000100:00000010:0.0:1775776358.523293:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2300. 00000100:00000001:1.0:1775776358.715289:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776358.715299:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776358.715302:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776358.715309:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776358.715314:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776358.715318:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776358.715320:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776358.715322:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776358.715325:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776358.843273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776358.843278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776358.843283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776358.843284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776358.843287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776358.843287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776358.843353:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776358.843357:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776358.843361:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776359.739266:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775776359.739267:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776359.739274:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776359.739278:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776359.739279:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776359.739284:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1775776359.739285:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:3.0:1775776359.739288:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776359.739291:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776359.739295:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776359.739297:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776359.739301:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776359.739304:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775776359.739306:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13242 00000100:00000001:1.0:1775776359.739309:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776359.739334:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776359.739343:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776359.739346:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776359.739350:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776359.739354:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776359.739356:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776359.739358:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776359.739363:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776359.739365:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776359.739368:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776359.739371:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776359.739374:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776359.739377:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776359.739380:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776359.739381:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776359.739383:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776359.739385:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776359.739389:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776359.739392:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776359.739395:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776359.739399:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776359.739403:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776359.739406:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776359.739409:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776359.739412:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776359.739424:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775776359.739428:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776359.739432:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776359.739434:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776359.739437:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776359.739440:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776359.739445:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776359.739455:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775776359.739464:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776359.739475:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710408448/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776359.739487:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776359.739494:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776359.739507:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710408448/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776359.739530:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776359.739533:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776359.739536:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776359.739537:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776359.739539:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776359.739542:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776359.739544:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13242 00000400:00000200:3.0:1775776359.739548:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776359.739552:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776359.739555:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776359.739558:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776359.739559:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776359.739561:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776359.739563:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775776359.739565:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776359.739566:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776359.739570:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:1.0:1775776359.739574:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71180 x1862022710408448/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775776359.739582:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775776359.739583:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776359.739585:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776359.739586:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 02000000:00000001:1.0:1775776359.739589:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1775776359.739590:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:1.0:1775776359.739592:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1775776359.739593:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 02000000:00000001:1.0:1775776359.739596:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775776359.739597:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00100000:1.0:1775776359.739600:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710408448:192.168.201.115@tcp:250:kworker.0 00000020:00000001:3.0:1775776359.739601:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776359.739605:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1775776359.739608:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776359.739609:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1775776359.739612:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775776359.739613:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776359.739616:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1775776359.739618:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776359.739621:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000010:1.0:1775776359.739622:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776359.739625:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776359.739625:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000400:00000200:1.0:1775776359.739631:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00080000:3.0:1775776359.739637:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776359.739639:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776359.739641:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 02000000:00000001:3.0:1775776359.739642:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776359.739649:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1775776359.739652:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 02000000:00000001:3.0:1775776359.739653:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776359.739657:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776359.739661:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000200:1.0:1775776359.739662:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710408448, portal 25 00000008:00000001:3.0:1775776359.739667:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776359.739668:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776359.739671:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849008512 : -123823860543104 : ffff8f6202c71180) 00000100:00000001:3.0:1775776359.739672:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776359.739680:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71180 x1862022710408448/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776384 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1775776359.739686:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775776359.739691:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000001:1.0:1775776359.739692:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1775776359.739694:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776359.739697:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776359.739698:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776359.739701:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:1.0:1775776359.739703:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 02000000:00000001:3.0:1775776359.739704:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:1.0:1775776359.739707:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710408448, offset 0 02000000:00000001:3.0:1775776359.739707:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776359.739715:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 02000000:00000010:3.0:1775776359.739715:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000001:3.0:1775776359.739720:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776359.739727:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00100000:3.0:1775776359.739728:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710408576/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776359.739734:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:3.0:1775776359.739740:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775776359.739741:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1775776359.739744:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776359.739748:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000040:3.0:1775776359.739752:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710408576/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776359.739760:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776359.739765:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:3.0:1775776359.739769:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776359.739770:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453743:453743:256:4294967295] 192.168.201.115@tcp LPNI seq info [453668:453668:8:4294967295] 00000100:00080000:3.0:1775776359.739773:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776359.739776:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00080000:3.0:1775776359.739780:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776359.739782:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776359.739784:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775776359.739790:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:3.0:1775776359.739791:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776359.739796:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13242 00000800:00000200:1.0:1775776359.739800:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000400:00000200:3.0:1775776359.739804:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000010:1.0:1775776359.739807:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000400:00000200:3.0:1775776359.739809:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776359.739813:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1775776359.739817:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776359.739818:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776359.739821:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.0:1775776359.739822:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00000040:3.0:1775776359.739824:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776359.739828:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000200:1.1:1775776359.739832:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776359.739835:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775776359.739839:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776359.739841:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000020:00000001:3.0:1775776359.739843:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776359.739845:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776359.739847:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:1.0:1775776359.739848:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:3.0:1775776359.739850:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000400:00000200:1.0:1775776359.739853:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000020:00000001:3.0:1775776359.739855:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776359.739858:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000020:00000001:3.0:1775776359.739859:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000010:1.0:1775776359.739862:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000040:3.0:1775776359.739865:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776359.739868:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776359.739870:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776359.739872:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:1.0:1775776359.739873:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:3.0:1775776359.739888:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00000200:1.0:1775776359.739890:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71180 x1862022710408448/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776384 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775776359.739895:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776359.739901:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776359.739904:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1775776359.739905:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1775776359.739909:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71180 x1862022710408448/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776384 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775776359.739909:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776359.739915:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776359.739919:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776359.739922:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775776359.739922:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775776359.739925:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:3.0:1775776359.739928:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776359.739931:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776359.739935:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776359.739935:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775776359.739940:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:1.0:1775776359.739943:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776359.739944:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776359.739947:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776359.739948:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307918598 00000100:00000001:3.0:1775776359.739949:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776359.739953:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776359.739955:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776359.739957:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2de0. 02000000:00000010:3.0:1775776359.739960:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:3.0:1775776359.739965:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776359.739966:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00100000:3.0:1775776359.739971:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710408704/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776359.739974:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307918598 00000100:00000010:1.0:1775776359.739981:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2660. 00000100:00000001:3.0:1775776359.739983:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776359.739985:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1775776359.739986:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776359.739989:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776359.739993:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776359.739993:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710408704/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.739996:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776359.740003:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776359/real 1775776359] req@ffff8f6202c71180 x1862022710408448/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776359.740010:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776359.740015:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776359.740023:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000001:1.0:1775776359.740027:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:3.0:1775776359.740028:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:1.0:1775776359.740030:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776359.740033:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000100:00000040:3.0:1775776359.740033:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1775776359.740038:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776359.740039:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000040:3.0:1775776359.740041:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13252) 00000100:00000001:1.0:1775776359.740042:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776359.740046:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71180 x1862022710408448/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776359.740052:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71180 x1862022710408448/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.740057:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776359.740058:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776359.740059:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776359.740061:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776359.740063:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71180 x1862022710408448/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.740067:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776359.740071:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71180 x1862022710408448/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.740076:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776359.740078:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776359.740080:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776359.740081:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776359.740083:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776359.740085:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776359.740088:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776359.740092:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776359.740094:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776359.740098:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71180 x1862022710408448/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776359.740107:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710408448:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776359.740112:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776359.740114:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776359.740117:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71180 x1862022710408448/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.740149:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776359.740152:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776359.740155:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776359.740157:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776359.740159:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776359.740160:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776359.740162:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776359.740163:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37400. 02000000:00000001:1.0:1775776359.740167:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776359.740169:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776359.740170:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71180. 00000100:00000001:1.0:1775776359.740173:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776359.740173:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776359.740175:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776359.740182:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776359.740183:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776359.740184:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776359.740185:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776359.740186:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776359.740189:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72a00 x1862022710408576/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.740194:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776359.740195:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776359.740196:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776359.740198:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776359.740199:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776359.740200:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710408576:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776359.740203:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776359.740204:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776359.740205:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776359.740206:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776359.740208:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776359.740209:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776359.740212:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776359.740214:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776359.740217:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776359.740219:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710408576, portal 10 00000100:00000001:1.0:1775776359.740220:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776359.740221:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849014784 : -123823860536832 : ffff8f6202c72a00) 00000100:00000040:1.0:1775776359.740226:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72a00 x1862022710408576/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776384 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.740230:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776359.740232:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776359.740233:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776359.740234:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710408576, offset 0 00000400:00000200:1.0:1775776359.740237:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776359.740239:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776359.740242:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776359.740245:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776359.740247:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776359.740250:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776359.740260:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776359.740266:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453744:453744:256:4294967295] 192.168.201.115@tcp LPNI seq info [453669:453669:8:4294967295] 00000400:00000200:1.0:1775776359.740271:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776359.740277:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776359.740282:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776359.740286:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776359.740290:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776359.740293:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775776359.740297:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776359.740299:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775776359.740301:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776359.740303:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776359.740304:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776359.740307:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776359.740309:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776359.740310:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776359.740312:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776359.740314:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72a00 x1862022710408576/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776384 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.740320:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776359.740321:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72a00 x1862022710408576/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776384 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.740325:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776359.740326:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776359.740329:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776359.740333:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776359.740336:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776359.740339:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307914764 00000100:00000001:1.0:1775776359.740342:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307918598 00000100:00000001:1.0:1775776359.740344:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307914764 00000100:00000001:1.0:1775776359.740346:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307918598 00000100:00000001:1.0:1775776359.740348:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776359.740350:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776359.740352:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776359.740355:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776359/real 1775776359] req@ffff8f6202c72a00 x1862022710408576/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776359.740363:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776359.740365:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776359.740367:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776359.740369:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776359.740371:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776359.740383:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776359.740386:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72a00 x1862022710408576/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776359.740394:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72a00 x1862022710408576/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.740400:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776359.740402:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776359.740404:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776359.740405:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776359.740408:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72a00 x1862022710408576/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.740413:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776359.740416:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72a00 x1862022710408576/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.740420:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776359.740421:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776359.740422:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776359.740423:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776359.740424:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776359.740428:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776359.740430:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776359.740432:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776359.740434:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776359.740436:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72a00 x1862022710408576/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776359.740441:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710408576:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776359.740444:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776359.740446:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72300 x1862022710408704/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.740450:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776359.740451:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776359.740452:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776359.740453:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776359.740455:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776359.740456:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710408704:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776359.740458:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776359.740459:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776359.740460:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776359.740461:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776359.740463:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776359.740464:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776359.740467:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776359.740468:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776359.740470:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776359.740472:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710408704, portal 10 00000100:00000001:1.0:1775776359.740474:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776359.740475:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849012992 : -123823860538624 : ffff8f6202c72300) 00000100:00000040:1.0:1775776359.740490:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72300 x1862022710408704/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776384 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.740496:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776359.740498:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776359.740501:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776359.740504:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710408704, offset 0 00000400:00000200:1.0:1775776359.740507:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776359.740511:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776359.740515:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776359.740517:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776359.740520:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776359.740523:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776359.740525:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776359.740528:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453745:453745:256:4294967295] 192.168.201.115@tcp LPNI seq info [453670:453670:8:4294967295] 00000400:00000200:1.0:1775776359.740532:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776359.740536:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776359.740539:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776359.740541:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776359.740544:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776359.740546:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775776359.740549:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776359.740550:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775776359.740553:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776359.740554:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776359.740556:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776359.740558:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776359.740560:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776359.740561:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776359.740563:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776359.740565:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72300 x1862022710408704/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776384 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.740570:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776359.740572:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72300 x1862022710408704/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776384 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.740576:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776359.740577:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776359.740579:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776359.740581:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776359.740583:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776359.740584:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307914764 00000100:00000001:1.0:1775776359.740586:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307918598 00000100:00000001:1.0:1775776359.740588:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307914764 00000100:00000001:1.0:1775776359.740589:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307918598 00000100:00000001:1.0:1775776359.740591:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776359.740592:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776359.740593:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776359.740595:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776359/real 1775776359] req@ffff8f6202c72300 x1862022710408704/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776359.740599:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776359.740600:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776359.740601:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776359.740602:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776359.740603:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776359.740604:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776359.740606:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72300 x1862022710408704/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776359.740610:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72300 x1862022710408704/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.740614:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776359.740615:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776359.740616:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776359.740616:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776359.740618:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72300 x1862022710408704/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.740621:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776359.740623:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72300 x1862022710408704/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.740627:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776359.740628:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776359.740629:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776359.740630:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776359.740631:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776359.740632:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776359.740634:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776359.740637:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776359.740638:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776359.740642:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72300 x1862022710408704/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776359.740649:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710408704:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776359.740652:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776359.740654:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776359.740656:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72a00 x1862022710408576/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.740662:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776359.740664:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776359.740666:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776359.740668:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776359.740669:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776359.740671:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776359.740673:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776359.740674:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35800. 02000000:00000001:1.0:1775776359.740677:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776359.740678:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776359.740679:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72a00. 00000100:00000001:1.0:1775776359.740681:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776359.740683:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776359.740684:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776359.740687:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72300 x1862022710408704/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776384 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776359.740692:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776359.740692:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776359.740693:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776359.740695:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776359.740696:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776359.740697:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776359.740698:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776359.740699:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35400. 02000000:00000001:1.0:1775776359.740700:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776359.740701:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776359.740701:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72300. 00000100:00000001:1.0:1775776359.740702:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776359.740703:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776359.740704:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776359.740706:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776359.740707:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776359.740708:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776359.740709:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776359.867281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776359.867287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776359.867292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776359.867293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776359.867297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776359.867297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776359.867403:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776359.867408:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776359.867411:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776360.763251:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776360.763259:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776360.763260:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776360.763266:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776360.763270:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776360.763274:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776360.763277:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776360.763280:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776360.763282:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776360.891275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776360.891280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776360.891284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776360.891288:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776360.891290:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776360.891292:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776360.891295:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776360.891297:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776360.891299:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776360.891300:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776360.891303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776360.891304:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776360.891305:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776360.891305:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776360.891306:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776360.891307:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776360.891308:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776360.891309:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776360.891365:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776360.891374:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776360.891378:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776361.787276:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776361.787284:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776361.787285:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776361.787292:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776361.787296:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776361.787301:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776361.787304:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776361.787306:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776361.787309:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776361.915249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776361.915255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776361.915261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776361.915263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776361.915273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776361.915274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776361.915325:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776361.915334:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776361.915337:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776362.811215:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776362.811222:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776362.811224:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776362.811229:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776362.811234:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776362.811239:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776362.811241:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776362.811244:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776362.811246:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776362.939307:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776362.939314:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776362.939322:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776362.939323:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776362.939327:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776362.939328:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776362.939388:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776362.939391:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776362.939400:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776363.643243:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307918598, at 4307918656 00000100:00000040:0.0:1775776363.643257:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307918598, at 4307918656 00000100:00000010:1.0:1775776363.643259:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2660. 00000100:00000010:0.0:1775776363.643268:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2de0. 00000100:00000001:1.0:1775776363.835231:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776363.835242:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776363.835244:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776363.835250:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776363.835253:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776363.835257:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776363.835259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776363.835261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776363.835263:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776363.963203:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776363.963208:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776363.963213:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776363.963215:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776363.963219:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776363.963220:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776363.963276:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776363.963278:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776363.963280:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776364.859200:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776364.859207:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776364.859209:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776364.859212:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776364.859214:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776364.859216:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776364.859220:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776364.859223:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776364.859224:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13247 00000100:00100000:1.0:1775776364.859229:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776364.859233:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1775776364.859234:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776364.859238:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1775776364.859241:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:1.0:1775776364.859243:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775776364.859244:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776364.859246:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776364.859249:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776364.859251:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776364.859252:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776364.859256:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776364.859258:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776364.859261:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776364.859262:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776364.859264:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776364.859266:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776364.859268:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776364.859270:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776364.859272:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776364.859274:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776364.859275:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776364.859276:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776364.859278:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776364.859280:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776364.859282:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776364.859283:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776364.859285:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776364.859287:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776364.859289:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776364.859290:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776364.859297:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73b80. 00000020:00000040:3.0:1775776364.859300:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776364.859303:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776364.859304:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776364.859305:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776364.859307:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776364.859310:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776364.859324:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:3.0:1775776364.859328:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776364.859336:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73b80 x1862022710408832/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776364.859343:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776364.859346:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776364.859349:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c52008f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73b80 x1862022710408832/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776364.859364:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776364.859365:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776364.859368:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776364.859369:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776364.859369:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776364.859371:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776364.859372:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13247 00000400:00000200:3.0:1775776364.859374:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776364.859377:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776364.859378:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776364.859380:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776364.859382:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776364.859384:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776364.859386:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776364.859388:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776364.859389:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:1.0:1775776364.859390:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1775776364.859391:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776364.859394:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:1.0:1775776364.859395:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1775776364.859396:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776364.859398:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1775776364.859398:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776364.859402:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776364.859403:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776364.859406:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000040:1.0:1775776364.859407:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73b80 x1862022710408832/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776364.859408:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776364.859411:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776364.859415:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776364.859419:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776364.859423:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776364.859427:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1775776364.859428:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776364.859430:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776364.859431:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776364.859435:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776364.859436:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:3.0:1775776364.859438:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775776364.859439:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1775776364.859442:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776364.859444:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776364.859444:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776364.859451:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775776364.859453:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776364.859454:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776364.859455:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776364.859456:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776364.859457:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776364.859459:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776364.859460:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710408832:192.168.201.115@tcp:250:kworker.0 02000000:00000010:3.0:1775776364.859463:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000100:00000001:1.0:1775776364.859468:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1775776364.859469:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776364.859474:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:3.0:1775776364.859474:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710408960/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1775776364.859478:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776364.859483:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775776364.859483:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776364.859486:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776364.859488:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710408960/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1775776364.859490:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776364.859496:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776364.859497:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776364.859501:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776364.859503:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776364.859505:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1775776364.859506:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00080000:3.0:1775776364.859507:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776364.859511:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776364.859513:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13247 00000400:00000010:1.0:1775776364.859514:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000200:3.0:1775776364.859517:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776364.859521:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776364.859522:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000010:1.0:1775776364.859523:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000040:3.0:1775776364.859524:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776364.859528:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776364.859529:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776364.859531:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776364.859532:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000200:1.0:1775776364.859533:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710408832, portal 25 00000020:00000040:3.0:1775776364.859533:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776364.859534:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776364.859536:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000100:00000001:1.0:1775776364.859539:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1775776364.859539:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000001:1.0:1775776364.859542:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849019264 : -123823860532352 : ffff8f6202c73b80) 00000020:00000001:3.0:1775776364.859542:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776364.859547:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776364.859549:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776364.859553:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000040:1.0:1775776364.859555:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73b80 x1862022710408832/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776389 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776364.859555:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776364.859558:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776364.859562:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776364.859566:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776364.859569:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776364.859570:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776364.859571:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776364.859572:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776364.859573:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776364.859574:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776364.859575:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776364.859577:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775776364.859580:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776364.859583:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1775776364.859584:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1775776364.859584:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776364.859587:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776364.859588:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776364.859590:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776364.859592:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 02000000:00000010:3.0:1775776364.859593:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000001:3.0:1775776364.859596:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776364.859597:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00100000:3.0:1775776364.859601:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710409088/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1775776364.859602:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710408832, offset 0 00000100:00000001:3.0:1775776364.859611:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776364.859612:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776364.859616:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776364.859618:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710409088/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776364.859623:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776364.859625:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776364.859627:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775776364.859628:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776364.859632:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776364.859634:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776364.859636:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13257) 00000400:00000200:1.0:1775776364.859637:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776364.859643:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776364.859648:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776364.859653:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776364.859656:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776364.859660:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453746:453746:256:4294967295] 192.168.201.115@tcp LPNI seq info [453671:453671:8:4294967295] 00000400:00000200:1.0:1775776364.859664:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776364.859674:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776364.859679:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776364.859685:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776364.859691:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776364.859695:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775776364.859701:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776364.859704:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1775776364.859708:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776364.859710:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776364.859715:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776364.859720:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776364.859725:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776364.859728:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776364.859732:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776364.859736:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73b80 x1862022710408832/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776389 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.859745:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776364.859749:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73b80 x1862022710408832/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776389 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.859756:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776364.859757:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776364.859762:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776364.859766:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776364.859769:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307923718 00000100:00000010:1.0:1775776364.859773:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2660. 00000100:00000001:1.0:1775776364.859779:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776364.859786:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307923718 00000100:00000010:1.0:1775776364.859813:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2b40. 00000100:00000001:1.0:1775776364.859817:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776364.859825:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776364.859828:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776364.859830:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776364.859834:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776364/real 1775776364] req@ffff8f6202c73b80 x1862022710408832/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776364.859844:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776364.859845:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776364.859847:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776364.859850:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776364.859852:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776364.859854:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776364.859858:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73b80 x1862022710408832/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776364.859867:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73b80 x1862022710408832/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.859874:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776364.859875:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776364.859878:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776364.859880:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776364.859883:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73b80 x1862022710408832/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.859890:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776364.859895:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73b80 x1862022710408832/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.859901:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776364.859904:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776364.859905:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776364.859907:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776364.859909:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776364.859911:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776364.859913:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776364.859916:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776364.859919:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776364.859923:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73b80 x1862022710408832/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776364.859931:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710408832:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776364.859935:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776364.859937:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776364.859940:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73b80 x1862022710408832/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.859947:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776364.859950:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776364.859952:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776364.859955:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776364.859956:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776364.859958:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776364.859960:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776364.859962:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37c00. 02000000:00000001:1.0:1775776364.859965:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776364.859967:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776364.859969:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73b80. 00000100:00000001:1.0:1775776364.859972:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776364.859973:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776364.859975:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776364.859982:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776364.859983:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776364.859985:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776364.859986:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776364.859987:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776364.859991:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72300 x1862022710408960/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.859998:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776364.859999:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776364.860001:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776364.860003:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776364.860005:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776364.860008:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710408960:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776364.860011:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776364.860013:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776364.860014:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776364.860016:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776364.860019:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776364.860021:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776364.860025:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776364.860027:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776364.860031:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776364.860033:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710408960, portal 10 00000100:00000001:1.0:1775776364.860035:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776364.860036:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849012992 : -123823860538624 : ffff8f6202c72300) 00000100:00000040:1.0:1775776364.860042:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72300 x1862022710408960/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776389 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.860049:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776364.860051:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776364.860053:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776364.860056:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710408960, offset 0 00000400:00000200:1.0:1775776364.860059:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776364.860064:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776364.860067:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776364.860069:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776364.860071:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776364.860073:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776364.860076:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776364.860078:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453747:453747:256:4294967295] 192.168.201.115@tcp LPNI seq info [453672:453672:8:4294967295] 00000400:00000200:1.0:1775776364.860082:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776364.860088:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776364.860092:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776364.860095:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776364.860099:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776364.860101:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775776364.860105:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776364.860108:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1775776364.860112:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776364.860113:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776364.860116:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776364.860119:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776364.860122:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776364.860124:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776364.860126:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776364.860130:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72300 x1862022710408960/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776389 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.860159:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776364.860162:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72300 x1862022710408960/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776389 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.860168:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776364.860170:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776364.860173:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776364.860177:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776364.860180:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776364.860181:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307919884 00000100:00000001:1.0:1775776364.860184:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307923718 00000100:00000001:1.0:1775776364.860186:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307919884 00000100:00000001:1.0:1775776364.860188:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307923718 00000100:00000001:1.0:1775776364.860190:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776364.860192:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776364.860194:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776364.860197:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776364/real 1775776364] req@ffff8f6202c72300 x1862022710408960/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776364.860211:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776364.860213:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776364.860215:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776364.860217:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776364.860219:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776364.860221:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776364.860224:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72300 x1862022710408960/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776364.860232:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72300 x1862022710408960/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.860239:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776364.860241:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776364.860242:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776364.860243:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776364.860246:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72300 x1862022710408960/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.860253:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776364.860256:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72300 x1862022710408960/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.860263:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776364.860265:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776364.860267:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776364.860268:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776364.860270:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776364.860274:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776364.860277:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776364.860281:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776364.860283:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776364.860288:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72300 x1862022710408960/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776364.860295:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710408960:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776364.860299:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776364.860303:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72a00 x1862022710409088/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.860309:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776364.860311:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776364.860313:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776364.860315:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776364.860317:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776364.860319:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710409088:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776364.860322:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776364.860324:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776364.860326:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776364.860327:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776364.860331:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776364.860333:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776364.860336:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776364.860339:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776364.860342:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776364.860344:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710409088, portal 10 00000100:00000001:1.0:1775776364.860346:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776364.860348:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849014784 : -123823860536832 : ffff8f6202c72a00) 00000100:00000040:1.0:1775776364.860353:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72a00 x1862022710409088/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776389 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.860359:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776364.860361:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776364.860364:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776364.860367:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710409088, offset 0 00000400:00000200:1.0:1775776364.860371:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776364.860375:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776364.860378:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776364.860381:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776364.860383:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776364.860385:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776364.860387:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776364.860390:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453748:453748:256:4294967295] 192.168.201.115@tcp LPNI seq info [453673:453673:8:4294967295] 00000400:00000200:1.0:1775776364.860395:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776364.860401:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776364.860405:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776364.860408:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776364.860411:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776364.860414:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775776364.860418:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776364.860420:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1775776364.860423:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776364.860425:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776364.860427:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776364.860430:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776364.860433:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776364.860435:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776364.860437:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776364.860440:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72a00 x1862022710409088/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776389 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.860447:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776364.860450:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72a00 x1862022710409088/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776389 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.860456:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776364.860457:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776364.860460:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776364.860463:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776364.860466:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776364.860468:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307919884 00000100:00000001:1.0:1775776364.860471:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307923718 00000100:00000001:1.0:1775776364.860473:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307919884 00000100:00000001:1.0:1775776364.860475:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307923718 00000100:00000001:1.0:1775776364.860477:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776364.860479:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776364.860481:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776364.860484:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776364/real 1775776364] req@ffff8f6202c72a00 x1862022710409088/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776364.860491:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776364.860493:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776364.860494:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776364.860496:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776364.860498:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776364.860500:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776364.860503:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72a00 x1862022710409088/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776364.860511:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72a00 x1862022710409088/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.860517:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776364.860518:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776364.860520:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776364.860521:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776364.860524:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72a00 x1862022710409088/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.860531:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776364.860534:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72a00 x1862022710409088/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.860541:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776364.860544:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776364.860545:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776364.860546:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776364.860548:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776364.860549:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776364.860552:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776364.860555:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776364.860557:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776364.860560:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72a00 x1862022710409088/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776364.860567:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710409088:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776364.860571:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776364.860573:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776364.860575:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72300 x1862022710408960/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.860582:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776364.860583:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776364.860585:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776364.860588:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776364.860589:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776364.860590:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776364.860592:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776364.860594:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35000. 02000000:00000001:1.0:1775776364.860596:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776364.860597:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776364.860599:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72300. 00000100:00000001:1.0:1775776364.860601:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776364.860602:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776364.860604:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776364.860607:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72a00 x1862022710409088/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776389 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776364.860618:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776364.860620:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776364.860621:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776364.860623:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776364.860624:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776364.860626:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776364.860627:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776364.860629:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37800. 02000000:00000001:1.0:1775776364.860630:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776364.860631:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776364.860633:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72a00. 00000100:00000001:1.0:1775776364.860634:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776364.860635:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776364.860637:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776364.860641:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776364.860642:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776364.860644:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776364.860645:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776364.987245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776364.987250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776364.987253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776364.987254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776364.987256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776364.987257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776364.987312:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776364.987314:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776364.987317:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776365.883214:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776365.883218:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776365.883220:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776365.883223:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776365.883225:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776365.883228:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776365.883230:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776365.883232:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776365.883233:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776366.011229:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776366.011236:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776366.011239:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776366.011243:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776366.011248:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776366.011250:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776366.011253:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776366.011255:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776366.011260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776366.011261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776366.011265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776366.011267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776366.011312:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776366.011316:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776366.011319:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776366.907248:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776366.907254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776366.907256:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776366.907262:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776366.907266:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776366.907270:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776366.907273:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776366.907275:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776366.907277:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776367.035237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776367.035242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776367.035247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776367.035249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776367.035252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776367.035253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776367.035309:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776367.035313:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776367.035316:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776367.931213:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776367.931219:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776367.931221:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776367.931226:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776367.931229:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776367.931232:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776367.931235:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776367.931237:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776367.931239:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776368.059236:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776368.059241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776368.059245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776368.059245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776368.059247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776368.059248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776368.059304:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776368.059307:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776368.059311:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776368.763226:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307923718, at 4307923776 00000100:00000010:1.0:1775776368.763238:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2b40. 00000100:00000040:0.0:1775776368.763258:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307923718, at 4307923776 00000100:00000010:0.0:1775776368.763266:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2660. 00000100:00000001:1.0:1775776368.955261:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776368.955267:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776368.955269:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776368.955274:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776368.955278:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776368.955281:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776368.955284:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776368.955286:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776368.955289:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776369.083239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776369.083244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776369.083248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776369.083250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776369.083253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776369.083254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776369.083325:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776369.083328:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776369.083332:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776369.979210:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776369.979218:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776369.979221:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776369.979227:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776369.979231:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776369.979236:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776369.979240:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776369.979243:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776369.979247:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776369.979258:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776369.979264:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776369.979267:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776369.979270:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776369.979275:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776369.979282:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13252 00000400:00000200:2.0:1775776369.979290:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776369.979296:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776369.979299:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776369.979302:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776369.979305:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776369.979307:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776369.979309:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776369.979312:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776369.979314:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776369.979317:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776369.979319:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776369.979322:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776369.979324:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776369.979327:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776369.979328:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776369.979330:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776369.979332:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776369.979335:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776369.979337:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776369.979340:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776369.979343:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776369.979345:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776369.979348:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776369.979350:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776369.979352:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776369.979360:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775776369.979364:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776369.979367:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776369.979369:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776369.979372:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776369.979375:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776369.979378:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776369.979387:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775776369.979393:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776369.979400:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710409216/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776369.979410:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776369.979414:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776369.979418:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710409216/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776369.979440:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776369.979443:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776369.979446:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776369.979453:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776369.979455:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776369.979458:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776369.979461:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13252 00000400:00000200:2.0:1775776369.979465:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776369.979468:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:1.0:1775776369.979469:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1775776369.979470:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776369.979472:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776369.979475:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776369.979475:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776369.979479:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776369.979483:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449180 x1862022710409216/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775776369.979484:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776369.979488:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776369.979492:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776369.979495:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:1.0:1775776369.979496:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1775776369.979498:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775776369.979499:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775776369.979500:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 02000000:00000001:1.0:1775776369.979502:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1775776369.979503:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776369.979505:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1775776369.979507:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776369.979508:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776369.979513:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00100000:1.0:1775776369.979514:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710409216:192.168.201.115@tcp:250:kworker.0 00000100:00000001:2.0:1775776369.979517:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776369.979519:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:2.0:1775776369.979522:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776369.979523:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1775776369.979524:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776369.979526:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775776369.979528:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:1.0:1775776369.979530:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1775776369.979532:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776369.979535:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776369.979536:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776369.979538:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776369.979538:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776369.979543:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775776369.979545:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:2.0:1775776369.979547:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776369.979549:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000001:2.0:1775776369.979550:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776369.979555:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000010:1.0:1775776369.979556:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776369.979564:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710409216, portal 25 00000100:00000001:1.0:1775776369.979567:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000008:00000001:2.0:1775776369.979567:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776369.979569:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662539648 : -123823047011968 : ffff8f6233449180) 00000100:00000001:2.0:1775776369.979571:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776369.979575:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449180 x1862022710409216/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776394 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1775776369.979576:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775776369.979581:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000001:1.0:1775776369.979585:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1775776369.979586:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1775776369.979590:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 02000000:00000001:2.0:1775776369.979590:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776369.979593:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:1.0:1775776369.979594:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 02000000:00000001:2.0:1775776369.979595:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776369.979598:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776369.979599:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710409216, offset 0 02000000:00000010:2.0:1775776369.979605:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000400:00000200:1.0:1775776369.979609:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:2.0:1775776369.979611:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776369.979615:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00100000:2.0:1775776369.979618:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710409344/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776369.979622:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776369.979630:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1775776369.979632:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775776369.979634:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:2.0:1775776369.979636:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776369.979639:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000040:2.0:1775776369.979643:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710409344/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776369.979645:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776369.979652:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453749:453749:256:4294967295] 192.168.201.115@tcp LPNI seq info [453674:453674:8:4294967295] 00000400:00000200:1.0:1775776369.979661:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:2.0:1775776369.979662:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776369.979665:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776369.979672:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776369.979675:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1775776369.979676:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00080000:2.0:1775776369.979677:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776369.979683:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000800:00000200:1.0:1775776369.979687:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00080000:2.0:1775776369.979687:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13252 00000400:00000200:2.0:1775776369.979693:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000010:1.0:1775776369.979695:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000400:00000200:2.0:1775776369.979697:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776369.979711:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776369.979715:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776369.979718:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.0:1775776369.979720:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775776369.979723:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:1.0:1775776369.979726:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000100:00000001:2.0:1775776369.979726:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776369.979730:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776369.979732:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776369.979735:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:1.1:1775776369.979736:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000040:2.0:1775776369.979737:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776369.979742:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:1.1:1775776369.979743:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000020:00000001:2.0:1775776369.979745:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000100:1.0:1775776369.979752:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000040:2.0:1775776369.979752:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000800:00000010:1.0:1775776369.979757:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000100:00000001:2.0:1775776369.979757:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776369.979761:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776369.979763:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776369.979765:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:2.0:1775776369.979766:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776369.979771:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000400:00000200:1.0:1775776369.979772:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1775776369.979777:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776369.979779:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 02000000:00000001:2.0:1775776369.979780:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:1.0:1775776369.979783:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 02000000:00000001:2.0:1775776369.979784:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776369.979788:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776369.979789:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1775776369.979791:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000200:1.0:1775776369.979794:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449180 x1862022710409216/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776394 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1775776369.979794:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776369.979801:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776369.979805:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776369.979810:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000100:00000001:1.0:1775776369.979811:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776369.979815:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449180 x1862022710409216/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776394 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1775776369.979815:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776369.979820:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776369.979823:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776369.979827:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775776369.979828:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776369.979829:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775776369.979830:0:2411:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1775776369.979832:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776369.979836:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776369.979837:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000400:00000200:1.0:1775776369.979841:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:2.0:1775776369.979842:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776369.979843:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307928838 00000100:00000010:1.0:1775776369.979850:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2b40. 00000100:00100000:2.0:1775776369.979850:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710409472/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776369.979856:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776369.979864:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307928838 00000100:00000010:1.0:1775776369.979866:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e20c0. 00000100:00000001:2.0:1775776369.979867:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776369.979868:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775776369.979870:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776369.979872:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776369.979874:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776369.979876:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710409472/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.979877:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776369.979883:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776369/real 1775776369] req@ffff8f6233449180 x1862022710409216/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776369.979896:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1775776369.979899:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:2.0:1775776369.979900:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:1.0:1775776369.979901:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776369.979904:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000100:00000040:2.0:1775776369.979905:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1775776369.979907:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776369.979909:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000040:2.0:1775776369.979909:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776369.979913:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:2.0:1775776369.979915:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000200:1.0:1775776369.979916:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449180 x1862022710409216/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775776369.979920:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13262) 00000100:00000200:1.0:1775776369.979924:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449180 x1862022710409216/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.979929:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776369.979930:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776369.979931:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776369.979932:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776369.979935:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449180 x1862022710409216/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.979939:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776369.979942:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449180 x1862022710409216/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.979945:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776369.979947:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776369.979948:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776369.979949:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776369.979951:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776369.979952:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776369.979954:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776369.979956:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776369.979958:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776369.979960:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449180 x1862022710409216/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776369.979964:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710409216:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776369.979967:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776369.979968:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776369.979969:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449180 x1862022710409216/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.979973:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776369.979974:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776369.979976:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776369.979978:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776369.979979:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776369.979980:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776369.979981:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776369.979982:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4f400. 02000000:00000001:1.0:1775776369.979984:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776369.979986:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776369.979987:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449180. 00000100:00000001:1.0:1775776369.979989:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776369.979990:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776369.979991:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776369.979996:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776369.979997:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776369.979998:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776369.979999:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776369.980000:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776369.980001:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449c00 x1862022710409344/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.980006:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776369.980007:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776369.980008:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776369.980009:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776369.980010:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776369.980012:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710409344:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776369.980013:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776369.980014:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776369.980015:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776369.980016:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776369.980018:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776369.980019:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776369.980022:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776369.980023:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776369.980025:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776369.980027:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710409344, portal 10 00000100:00000001:1.0:1775776369.980028:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776369.980029:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662542336 : -123823047009280 : ffff8f6233449c00) 00000100:00000040:1.0:1775776369.980033:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449c00 x1862022710409344/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776394 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.980036:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776369.980038:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776369.980039:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776369.980040:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710409344, offset 0 00000400:00000200:1.0:1775776369.980042:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776369.980044:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776369.980046:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776369.980048:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776369.980050:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776369.980051:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776369.980052:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776369.980055:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453750:453750:256:4294967295] 192.168.201.115@tcp LPNI seq info [453675:453675:8:4294967295] 00000400:00000200:1.0:1775776369.980057:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776369.980062:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776369.980064:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776369.980066:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776369.980068:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776369.980070:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775776369.980073:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776369.980074:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1775776369.980077:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776369.980078:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776369.980080:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776369.980082:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776369.980083:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776369.980084:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776369.980086:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776369.980088:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449c00 x1862022710409344/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776394 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.980093:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776369.980095:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449c00 x1862022710409344/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776394 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.980098:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776369.980099:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776369.980101:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776369.980103:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776369.980105:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776369.980106:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307925003 00000100:00000001:1.0:1775776369.980107:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307928838 00000100:00000001:1.0:1775776369.980109:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307925003 00000100:00000001:1.0:1775776369.980110:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307928838 00000100:00000001:1.0:1775776369.980112:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776369.980113:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776369.980114:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776369.980116:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776369/real 1775776369] req@ffff8f6233449c00 x1862022710409344/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776369.980121:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776369.980122:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776369.980123:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776369.980124:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776369.980125:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776369.980126:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776369.980128:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449c00 x1862022710409344/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776369.980150:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449c00 x1862022710409344/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.980154:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776369.980155:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776369.980156:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776369.980157:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776369.980159:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449c00 x1862022710409344/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.980163:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776369.980166:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449c00 x1862022710409344/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.980170:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776369.980171:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776369.980172:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776369.980172:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776369.980173:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776369.980176:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776369.980178:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776369.980180:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776369.980181:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776369.980184:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449c00 x1862022710409344/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776369.980188:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710409344:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776369.980190:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776369.980191:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233448700 x1862022710409472/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.980195:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776369.980195:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776369.980196:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776369.980197:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776369.980199:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776369.980200:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710409472:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776369.980203:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776369.980204:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776369.980206:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776369.980208:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776369.980210:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776369.980212:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776369.980215:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776369.980217:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776369.980220:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776369.980222:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710409472, portal 10 00000100:00000001:1.0:1775776369.980224:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776369.980225:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662536960 : -123823047014656 : ffff8f6233448700) 00000100:00000040:1.0:1775776369.980230:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233448700 x1862022710409472/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776394 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.980236:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776369.980239:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776369.980241:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776369.980243:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710409472, offset 0 00000400:00000200:1.0:1775776369.980247:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776369.980250:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776369.980253:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776369.980256:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776369.980258:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776369.980260:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776369.980261:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776369.980263:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453751:453751:256:4294967295] 192.168.201.115@tcp LPNI seq info [453676:453676:8:4294967295] 00000400:00000200:1.0:1775776369.980266:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776369.980272:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776369.980276:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776369.980286:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776369.980290:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776369.980293:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775776369.980296:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776369.980299:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1775776369.980303:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776369.980305:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776369.980307:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776369.980311:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776369.980313:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776369.980315:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776369.980318:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776369.980321:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233448700 x1862022710409472/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776394 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.980328:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776369.980331:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448700 x1862022710409472/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776394 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.980337:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776369.980339:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776369.980343:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776369.980345:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776369.980349:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776369.980351:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307925004 00000100:00000001:1.0:1775776369.980354:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307928838 00000100:00000001:1.0:1775776369.980355:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307925004 00000100:00000001:1.0:1775776369.980356:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307928838 00000100:00000001:1.0:1775776369.980358:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776369.980359:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776369.980361:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776369.980363:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776369/real 1775776369] req@ffff8f6233448700 x1862022710409472/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776369.980367:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776369.980368:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776369.980369:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776369.980371:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776369.980372:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776369.980374:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776369.980375:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233448700 x1862022710409472/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776369.980380:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233448700 x1862022710409472/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.980384:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776369.980384:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776369.980385:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776369.980386:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776369.980388:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233448700 x1862022710409472/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.980391:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776369.980394:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233448700 x1862022710409472/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.980398:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776369.980400:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776369.980401:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776369.980401:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776369.980402:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776369.980403:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776369.980404:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776369.980406:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776369.980407:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776369.980409:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233448700 x1862022710409472/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776369.980413:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710409472:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776369.980416:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776369.980417:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776369.980419:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449c00 x1862022710409344/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.980422:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776369.980423:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776369.980424:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776369.980426:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776369.980426:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776369.980427:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776369.980428:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776369.980429:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4c000. 02000000:00000001:1.0:1775776369.980431:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776369.980432:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776369.980433:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449c00. 00000100:00000001:1.0:1775776369.980434:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776369.980435:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776369.980436:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776369.980438:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233448700 x1862022710409472/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776394 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776369.980441:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776369.980442:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776369.980443:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776369.980444:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776369.980444:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776369.980445:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776369.980446:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776369.980447:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4e800. 02000000:00000001:1.0:1775776369.980448:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776369.980448:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776369.980449:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233448700. 00000100:00000001:1.0:1775776369.980450:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776369.980451:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776369.980452:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776369.980454:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776369.980454:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776369.980455:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776369.980456:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776370.107272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776370.107278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776370.107283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776370.107284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776370.107287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776370.107287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776370.107388:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776370.107393:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776370.107397:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776371.003265:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776371.003274:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776371.003276:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776371.003284:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776371.003289:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776371.003294:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776371.003297:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776371.003301:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776371.003305:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776371.131205:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776371.131209:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776371.131213:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776371.131216:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776371.131217:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776371.131220:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776371.131221:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776371.131223:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776371.131224:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776371.131226:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776371.131228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776371.131229:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776371.131230:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776371.131231:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776371.131232:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776371.131232:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776371.131233:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776371.131234:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776371.131284:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776371.131287:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776371.131290:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776372.027235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776372.027246:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776372.027247:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776372.027251:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776372.027254:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776372.027257:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776372.027259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776372.027261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776372.027263:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776372.155208:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776372.155212:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776372.155216:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776372.155217:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776372.155219:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776372.155220:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776372.155290:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776372.155294:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776372.155297:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776373.051231:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776373.051236:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776373.051238:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776373.051241:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776373.051244:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776373.051246:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776373.051248:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776373.051250:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776373.051251:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776373.179219:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776373.179223:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776373.179226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776373.179227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776373.179229:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776373.179230:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776373.179290:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776373.179294:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776373.179297:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776373.883217:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307928838, at 4307928896 00000800:00000200:0.0:1775776373.883228:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:0.0:1775776373.883233:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000100:00000010:1.0:1775776373.883233:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e20c0. 00000800:00000200:0.0:1775776373.883239:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000100:00000040:0.0:1775776373.883263:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307928838, at 4307928896 00000100:00000010:0.0:1775776373.883268:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2b40. 00000800:00000001:0.0:1775776373.883353:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776373.883364:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775776373.883369:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776373.883373:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1775776373.883373:0:2404:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1775776374.075230:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776374.075236:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776374.075238:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776374.075243:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776374.075247:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776374.075250:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776374.075253:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776374.075256:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776374.075258:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776374.203251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776374.203256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776374.203261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776374.203262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776374.203267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776374.203268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776374.203327:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776374.203330:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776374.203332:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776375.099225:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776375.099230:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776375.099232:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776375.099235:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776375.099239:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776375.099242:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775776375.099244:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776375.099244:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776375.099247:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776375.099251:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776375.099252:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776375.099256:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776375.099259:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776375.099263:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776375.099270:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13257 00000400:00000200:0.0:1775776375.099278:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776375.099282:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776375.099292:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776375.099295:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776375.099297:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776375.099299:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776375.099301:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776375.099304:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776375.099306:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776375.099308:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776375.099311:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776375.099313:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776375.099315:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776375.099318:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776375.099320:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776375.099322:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776375.099324:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776375.099327:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776375.099330:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776375.099332:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776375.099335:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776375.099339:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776375.099341:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776375.099344:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776375.099346:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776375.099355:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062d80. 00000020:00000040:0.0:1775776375.099357:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776375.099360:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776375.099361:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776375.099363:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776375.099365:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776375.099369:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776375.099375:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4800. 00000100:00000001:0.0:1775776375.099379:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776375.099386:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062d80 x1862022710409600/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776375.099394:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776375.099397:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776375.099400:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009965f80b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062d80 x1862022710409600/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776375.099414:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776375.099417:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776375.099419:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776375.099420:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776375.099421:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776375.099423:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776375.099425:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13257 00000400:00000200:0.0:1775776375.099429:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776375.099431:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:1.0:1775776375.099431:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776375.099434:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1775776375.099435:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776375.099437:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776375.099439:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776375.099444:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776375.099444:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e062d80 x1862022710409600/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776375.099448:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776375.099453:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776375.099458:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775776375.099459:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:0.0:1775776375.099460:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776375.099463:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775776375.099463:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775776375.099467:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 02000000:00000001:1.0:1775776375.099468:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776375.099471:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776375.099474:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776375.099479:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00100000:1.0:1775776375.099479:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e062d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710409600:192.168.201.115@tcp:250:kworker.0 00000020:00000001:0.0:1775776375.099484:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776375.099486:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1775776375.099490:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776375.099492:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775776375.099492:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:0.0:1775776375.099495:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776375.099496:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776375.099497:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776375.099499:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:0.0:1775776375.099500:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775776375.099503:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776375.099504:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776375.099505:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:1.0:1775776375.099506:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:0.0:1775776375.099509:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776375.099510:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776375.099514:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776375.099516:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775776375.099519:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000008:00100000:0.0:1775776375.099520:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000010:1.0:1775776375.099523:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000008:00000001:0.0:1775776375.099525:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776375.099528:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776375.099531:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000010:0.0:1775776375.099532:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062a00. 00000020:00000040:0.0:1775776375.099536:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776375.099538:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776375.099540:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776375.099540:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710409600, portal 25 00000100:00000001:0.0:1775776375.099544:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775776375.099545:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1775776375.099546:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775776375.099549:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574564736 : -123823134986880 : ffff8f622e062d80) 02000000:00000001:0.0:1775776375.099550:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776375.099557:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6000. 00000100:00000040:1.0:1775776375.099559:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e062d80 x1862022710409600/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776375.099562:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776375.099571:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062a00 x1862022710409728/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776375.099572:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776375.099578:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776375.099584:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000001:0.0:1775776375.099585:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776375.099588:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776375.099589:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710409600, offset 0 00000100:00000040:0.0:1775776375.099592:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009753f9f6] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062a00 x1862022710409728/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776375.099598:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776375.099600:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00080000:0.0:1775776375.099601:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776375.099605:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776375.099607:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776375.099608:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776375.099624:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776375.099626:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13257 00000400:00000200:1.0:1775776375.099627:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776375.099635:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776375.099640:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776375.099643:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776375.099647:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776375.099649:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776375.099653:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453752:453752:256:4294967295] 192.168.201.115@tcp LPNI seq info [453677:453677:8:4294967295] 00000400:00000200:1.0:1775776375.099658:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776375.099666:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000400:00000200:0.0:1775776375.099672:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000200:1.0:1775776375.099672:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776375.099677:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000800:00000010:1.0:1775776375.099678:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000100:00080000:0.0:1775776375.099679:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776375.099683:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776375.099685:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776375.099688:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:1.0:1775776375.099688:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775776375.099690:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776375.099692:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776375.099693:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000800:00000200:1.0:1775776375.099693:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775776375.099698:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:0.0:1775776375.099701:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:1.1:1775776375.099702:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000020:00000040:0.0:1775776375.099703:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775776375.099707:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000100:1.0:1775776375.099710:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000020:00000001:0.0:1775776375.099711:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000010:1.0:1775776375.099714:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000100:00000040:0.0:1775776375.099716:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776375.099718:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776375.099719:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776375.099720:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1775776375.099721:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776375.099723:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776375.099727:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00080000:0.0:1775776375.099728:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776375.099730:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776375.099731:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 02000000:00000001:0.0:1775776375.099732:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:1.0:1775776375.099735:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 02000000:00000001:0.0:1775776375.099736:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776375.099738:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776375.099739:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776375.099740:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:0.0:1775776375.099741:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776375.099743:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776375.099746:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776375.099747:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e062d80 x1862022710409600/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775776375.099751:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061500. 00000020:00000040:0.0:1775776375.099754:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776375.099758:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776375.099759:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776375.099760:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1775776375.099761:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776375.099764:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1775776375.099764:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e062d80 x1862022710409600/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776400 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775776375.099766:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776375.099771:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7800. 00000100:00000001:0.0:1775776375.099775:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776375.099778:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776375.099780:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:0.0:1775776375.099783:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061500 x1862022710409856/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776375.099785:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776375.099791:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776375.099795:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307933958 00000100:00000001:0.0:1775776375.099798:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000010:1.0:1775776375.099800:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e20c0. 00000100:00000001:0.0:1775776375.099801:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776375.099806:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000040:0.0:1775776375.099807:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb8d992] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061500 x1862022710409856/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.099813:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307933958 00000100:00000010:1.0:1775776375.099815:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2ba0. 00000100:00000001:1.0:1775776375.099817:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776375.099821:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776375.099825:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776375.099828:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1775776375.099833:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776375.099833:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776375/real 1775776375] req@ffff8f622e062d80 x1862022710409600/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776375.099835:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776375.099841:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776375.099845:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776375.099848:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000001:1.0:1775776375.099848:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776375.099852:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:0.0:1775776375.099853:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13267) 00000400:00000010:1.0:1775776375.099854:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776375.099858:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776375.099860:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776375.099863:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776375.099867:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e062d80 x1862022710409600/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776375.099877:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e062d80 x1862022710409600/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.099884:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776375.099885:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776375.099888:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776375.099890:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776375.099893:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e062d80 x1862022710409600/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.099900:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776375.099904:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e062d80 x1862022710409600/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.099911:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776375.099915:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776375.099917:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776375.099918:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776375.099920:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776375.099922:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776375.099925:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776375.099929:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776375.099931:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776375.099935:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e062d80 x1862022710409600/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776375.099943:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e062d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710409600:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776375.099948:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776375.099950:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776375.099953:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e062d80 x1862022710409600/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.099959:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776375.099961:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776375.099963:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776375.099965:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776375.099967:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776375.099968:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776375.099970:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776375.099972:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:1.0:1775776375.099976:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776375.099977:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776375.099979:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e062d80. 00000100:00000001:1.0:1775776375.099982:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776375.099983:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776375.099985:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776375.099991:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776375.099992:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776375.099994:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776375.099996:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776375.099998:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776375.100001:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e062a00 x1862022710409728/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.100007:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776375.100009:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776375.100010:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776375.100012:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776375.100014:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776375.100022:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e062a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710409728:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776375.100025:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776375.100027:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776375.100028:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776375.100030:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776375.100033:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776375.100035:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776375.100039:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776375.100041:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776375.100044:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776375.100047:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710409728, portal 10 00000100:00000001:1.0:1775776375.100049:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776375.100050:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574563840 : -123823134987776 : ffff8f622e062a00) 00000100:00000040:1.0:1775776375.100056:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e062a00 x1862022710409728/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.100062:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776375.100064:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776375.100066:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776375.100068:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710409728, offset 0 00000400:00000200:1.0:1775776375.100072:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776375.100076:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776375.100080:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776375.100083:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776375.100085:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776375.100087:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776375.100090:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776375.100092:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453753:453753:256:4294967295] 192.168.201.115@tcp LPNI seq info [453678:453678:8:4294967295] 00000400:00000200:1.0:1775776375.100096:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776375.100102:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776375.100106:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776375.100108:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776375.100111:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776375.100112:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775776375.100115:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776375.100117:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1775776375.100119:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776375.100120:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776375.100122:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776375.100124:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776375.100126:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776375.100127:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776375.100129:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776375.100131:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e062a00 x1862022710409728/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.100150:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776375.100152:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e062a00 x1862022710409728/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776400 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.100156:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776375.100157:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776375.100160:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776375.100163:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776375.100165:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776375.100166:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307930124 00000100:00000001:1.0:1775776375.100168:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307933958 00000100:00000001:1.0:1775776375.100170:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307930124 00000100:00000001:1.0:1775776375.100172:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307933958 00000100:00000001:1.0:1775776375.100173:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776375.100174:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776375.100175:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776375.100177:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776375/real 1775776375] req@ffff8f622e062a00 x1862022710409728/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776375.100182:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776375.100182:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776375.100183:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776375.100184:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776375.100185:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776375.100186:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776375.100188:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e062a00 x1862022710409728/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776375.100192:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e062a00 x1862022710409728/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.100196:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776375.100197:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776375.100198:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776375.100199:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776375.100200:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e062a00 x1862022710409728/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.100204:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776375.100206:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e062a00 x1862022710409728/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.100209:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776375.100210:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776375.100211:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776375.100212:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776375.100213:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776375.100217:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776375.100218:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776375.100220:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776375.100222:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776375.100224:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e062a00 x1862022710409728/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776375.100228:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e062a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710409728:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776375.100230:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776375.100232:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061500 x1862022710409856/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.100235:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776375.100236:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776375.100237:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776375.100238:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776375.100239:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776375.100241:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710409856:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776375.100243:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776375.100244:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776375.100245:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776375.100246:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776375.100248:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776375.100249:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776375.100251:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776375.100252:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776375.100254:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776375.100256:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710409856, portal 10 00000100:00000001:1.0:1775776375.100257:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776375.100258:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574558464 : -123823134993152 : ffff8f622e061500) 00000100:00000040:1.0:1775776375.100261:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061500 x1862022710409856/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.100265:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776375.100266:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776375.100267:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776375.100268:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710409856, offset 0 00000400:00000200:1.0:1775776375.100270:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776375.100272:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776375.100274:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776375.100276:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776375.100277:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776375.100279:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776375.100280:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776375.100282:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453754:453754:256:4294967295] 192.168.201.115@tcp LPNI seq info [453679:453679:8:4294967295] 00000400:00000200:1.0:1775776375.100284:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776375.100288:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776375.100291:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776375.100292:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776375.100294:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776375.100296:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775776375.100298:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776375.100300:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1775776375.100302:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776375.100303:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776375.100305:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776375.100307:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776375.100308:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776375.100309:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776375.100310:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776375.100312:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061500 x1862022710409856/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.100317:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776375.100318:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061500 x1862022710409856/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776400 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.100326:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776375.100327:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776375.100328:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776375.100330:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776375.100331:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776375.100332:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307930124 00000100:00000001:1.0:1775776375.100334:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307933958 00000100:00000001:1.0:1775776375.100335:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307930124 00000100:00000001:1.0:1775776375.100336:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307933958 00000100:00000001:1.0:1775776375.100337:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776375.100338:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776375.100339:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776375.100341:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776375/real 1775776375] req@ffff8f622e061500 x1862022710409856/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776375.100344:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776375.100345:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776375.100346:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776375.100347:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776375.100348:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776375.100349:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776375.100351:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061500 x1862022710409856/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776375.100355:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061500 x1862022710409856/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.100358:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776375.100359:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776375.100360:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776375.100360:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776375.100362:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061500 x1862022710409856/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.100366:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776375.100368:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061500 x1862022710409856/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.100372:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776375.100373:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776375.100374:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776375.100374:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776375.100375:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776375.100376:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776375.100377:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776375.100379:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776375.100380:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776375.100382:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061500 x1862022710409856/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776375.100386:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710409856:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776375.100388:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776375.100389:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776375.100390:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e062a00 x1862022710409728/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.100393:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776375.100394:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776375.100395:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:1.0:1775776375.100396:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776375.100397:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776375.100398:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776375.100399:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776375.100400:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:1.0:1775776375.100402:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776375.100403:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776375.100404:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e062a00. 00000100:00000001:1.0:1775776375.100405:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776375.100406:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776375.100407:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776375.100409:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061500 x1862022710409856/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776400 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776375.100412:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776375.100413:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776375.100413:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776375.100415:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776375.100415:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776375.100416:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776375.100417:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776375.100417:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:1.0:1775776375.100418:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776375.100419:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776375.100420:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061500. 00000100:00000001:1.0:1775776375.100421:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776375.100421:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776375.100422:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776375.100424:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776375.100425:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776375.100426:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776375.100427:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776375.227232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776375.227237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776375.227241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776375.227242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776375.227244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776375.227245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776375.227290:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776375.227294:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776375.227296:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776376.123250:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776376.123259:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776376.123261:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776376.123268:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776376.123273:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776376.123278:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776376.123281:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776376.123285:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776376.123288:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776376.251249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776376.251257:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776376.251259:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776376.251263:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776376.251267:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776376.251270:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776376.251271:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776376.251273:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776376.251277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776376.251279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776376.251282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776376.251284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776376.251330:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776376.251334:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776376.251336:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776377.147248:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776377.147254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776377.147256:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776377.147261:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776377.147265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776377.147268:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776377.147270:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776377.147272:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776377.147274:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776377.275282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776377.275288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776377.275294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776377.275295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776377.275298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776377.275300:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776377.275363:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776377.275367:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776377.275370:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776378.171211:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776378.171215:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776378.171217:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776378.171222:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776378.171226:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776378.171229:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776378.171231:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776378.171233:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776378.171235:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776378.299250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776378.299255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776378.299258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776378.299259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776378.299262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776378.299263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776378.299326:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776378.299330:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776378.299333:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776379.003234:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307933958, at 4307934016 00000100:00000010:1.0:1775776379.003250:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ba0. 00000100:00000040:0.0:1775776379.003258:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307933958, at 4307934016 00000100:00000010:0.0:1775776379.003267:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e20c0. 00000100:00000001:1.0:1775776379.195230:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776379.195237:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776379.195239:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776379.195243:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776379.195247:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776379.195250:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776379.195253:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776379.195255:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776379.195257:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776379.323205:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776379.323209:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776379.323212:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776379.323213:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776379.323216:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776379.323216:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776379.323263:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776379.323271:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776379.323273:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776380.219235:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776380.219241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776380.219245:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776380.219248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776380.219249:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776380.219253:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1775776380.219254:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776380.219258:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1775776380.219261:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776380.219262:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776380.219268:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776380.219268:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775776380.219270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776380.219274:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776380.219284:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13262 00000400:00000200:1.0:1775776380.219296:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776380.219304:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776380.219307:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776380.219310:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776380.219313:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776380.219315:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776380.219318:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:1.0:1775776380.219321:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776380.219323:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:1.0:1775776380.219326:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776380.219329:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:1.0:1775776380.219332:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:1.0:1775776380.219335:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:1.0:1775776380.219337:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776380.219339:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776380.219341:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776380.219342:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:1.0:1775776380.219345:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:1.0:1775776380.219347:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776380.219349:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776380.219353:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776380.219356:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776380.219359:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776380.219361:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776380.219364:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776380.219374:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6234919f80. 00000020:00000040:1.0:1775776380.219378:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:1.0:1775776380.219382:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776380.219384:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776380.219387:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776380.219390:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776380.219395:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776380.219403:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4d800. 00000100:00000001:1.0:1775776380.219410:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776380.219419:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6234919f80 x1862022710409984/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776380.219430:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776380.219435:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776380.219439:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0867d60] to pc [ptlrpcd_rcv+-1] req@ffff8f6234919f80 x1862022710409984/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776380.219460:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776380.219463:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776380.219466:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776380.219468:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776380.219470:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776380.219473:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776380.219475:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13262 00000100:00000001:0.0:1775776380.219477:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1775776380.219480:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776380.219481:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776380.219484:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000400:00000200:1.0:1775776380.219485:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776380.219488:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776380.219492:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234919f80 x1862022710409984/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776380.219495:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776380.219499:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776380.219504:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776380.219506:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776380.219507:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:0.0:1775776380.219508:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776380.219512:0:191261:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000001:0.0:1775776380.219514:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776380.219518:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776380.219530:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775776380.219535:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00100000:0.0:1775776380.219537:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234919f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710409984:192.168.201.115@tcp:250:kworker.0 00000020:00000001:1.0:1775776380.219538:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776380.219543:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:0.0:1775776380.219545:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:1.0:1775776380.219546:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:1.0:1775776380.219550:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:0.0:1775776380.219551:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:1.0:1775776380.219555:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 02000000:00000001:0.0:1775776380.219556:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776380.219557:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775776380.219560:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1775776380.219560:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776380.219563:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:1.0:1775776380.219565:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000010:0.0:1775776380.219567:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 00000100:00080000:1.0:1775776380.219568:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000001:0.0:1775776380.219571:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776380.219572:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776380.219576:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1775776380.219581:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:1.0:1775776380.219582:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776380.219586:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776380.219587:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000100:00000001:1.0:1775776380.219588:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776380.219595:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000010:0.0:1775776380.219597:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000008:00000001:1.0:1775776380.219599:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776380.219602:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776380.219605:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6234919c00. 00000100:00000200:0.0:1775776380.219606:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710409984, portal 25 00000020:00000040:1.0:1775776380.219609:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000001:0.0:1775776380.219611:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1775776380.219613:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1775776380.219614:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684366720 : -123823025184896 : ffff8f6234919f80) 02000000:00000001:1.0:1775776380.219616:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776380.219619:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776380.219622:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:0.0:1775776380.219623:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234919f80 x1862022710409984/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775776380.219624:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776380.219631:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4c800. 00000100:00000001:1.0:1775776380.219636:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776380.219637:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776380.219642:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00100000:1.0:1775776380.219646:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6234919c00 x1862022710410112/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1775776380.219647:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0198. 00000100:00000200:0.0:1775776380.219653:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710409984, offset 0 00000100:00000001:1.0:1775776380.219660:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776380.219663:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776380.219664:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000040:1.0:1775776380.219669:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000718661bc] to pc [ptlrpcd_rcv+-1] req@ffff8f6234919c00 x1862022710410112/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776380.219676:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776380.219687:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:1.0:1775776380.219691:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776380.219695:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:1.0:1775776380.219695:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776380.219700:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:0.0:1775776380.219703:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:1.0:1775776380.219704:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776380.219706:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1775776380.219710:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1775776380.219710:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776380.219716:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13262 00000400:00000200:0.0:1775776380.219717:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776380.219724:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453755:453755:256:4294967295] 192.168.201.115@tcp LPNI seq info [453680:453680:8:4294967295] 00000400:00000200:0.0:1775776380.219728:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776380.219738:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000400:00000200:1.0:1775776380.219745:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000200:0.0:1775776380.219746:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776380.219751:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000800:00000010:0.0:1775776380.219754:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cd00. 00000100:00080000:1.0:1775776380.219754:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776380.219759:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776380.219761:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776380.219765:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:0.0:1775776380.219767:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:1.0:1775776380.219768:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:1.0:1775776380.219773:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776380.219774:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:1.0:1775776380.219777:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:0.1:1775776380.219778:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000040:1.0:1775776380.219779:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:1.0:1775776380.219783:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:1.0:1775776380.219785:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:1.0:1775776380.219788:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776380.219790:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776380.219792:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776380.219793:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:1.0:1775776380.219795:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00000001:0.0:1775776380.219797:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776380.219798:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776380.219802:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776380.219806:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:1.0:1775776380.219807:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776380.219811:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776380.219812:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307939078 02000000:00000001:1.0:1775776380.219815:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776380.219818:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000010:0.0:1775776380.219820:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3480. 00000008:00100000:1.0:1775776380.219820:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776380.219825:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776380.219827:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776380.219831:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000010:1.0:1775776380.219831:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623491a300. 00000020:00000040:1.0:1775776380.219836:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776380.219838:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1775776380.219840:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307939078 02000000:00000001:1.0:1775776380.219840:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776380.219843:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776380.219845:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000010:0.0:1775776380.219846:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3a20. 02000000:00000001:1.0:1775776380.219847:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776380.219853:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 02000000:00000010:1.0:1775776380.219854:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4c000. 00000100:00000001:0.0:1775776380.219858:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776380.219859:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776380.219864:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776380.219868:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776380.219868:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623491a300 x1862022710410240/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776380.219873:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776380.219880:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776380.219882:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776380.219883:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776380.219886:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000040:1.0:1775776380.219887:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bfa067b] to pc [ptlrpcd_rcv+-1] req@ffff8f623491a300 x1862022710410240/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776380.219891:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776380.219893:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776380.219896:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1775776380.219902:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776380.219905:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234919c00 x1862022710410112/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776380.219906:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1775776380.219912:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:1.0:1775776380.219917:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776380.219922:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1775776380.219923:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1775776380.219926:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776380.219929:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1775776380.219930:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13272) 02000000:00000001:0.0:1775776380.219931:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776380.219935:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776380.219939:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234919c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710410112:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776380.219942:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776380.219944:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776380.219947:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776380.219949:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776380.219953:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775776380.219955:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776380.219959:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776380.219963:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776380.219967:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776380.219970:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710410112, portal 10 00000100:00000001:0.0:1775776380.219972:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776380.219973:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684365824 : -123823025185792 : ffff8f6234919c00) 00000100:00000040:0.0:1775776380.219981:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234919c00 x1862022710410112/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776380.219988:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776380.219991:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776380.219994:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0cc0. 00000100:00000200:0.0:1775776380.219997:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710410112, offset 0 00000400:00000200:0.0:1775776380.220002:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776380.220005:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776380.220009:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776380.220012:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776380.220014:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776380.220016:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776380.220018:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776380.220022:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453756:453756:255:4294967295] 192.168.201.115@tcp LPNI seq info [453681:453681:7:4294967295] 00000400:00000200:0.0:1775776380.220026:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776380.220032:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776380.220037:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776380.220041:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c600. 00000800:00000200:0.0:1775776380.220044:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776380.220048:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775776380.220052:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776380.220056:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776380.220059:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776380.220061:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307935243 00000100:00000001:0.0:1775776380.220064:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307939078 00000100:00000001:0.0:1775776380.220066:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307935243 00000100:00000001:0.0:1775776380.220068:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307939078 00000100:00000001:0.0:1775776380.220070:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776380.220072:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776380.220075:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776380.220076:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776380.220079:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776380.220081:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1775776380.220083:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776380.220084:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776380.220085:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776380.220089:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491a300 x1862022710410240/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776380.220096:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776380.220098:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776380.220100:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776380.220101:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776380.220103:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776380.220106:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710410240:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776380.220109:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776380.220111:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776380.220112:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776380.220114:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776380.220123:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776380.220125:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776380.220128:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776380.220131:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d68. 00000400:00000010:0.0:1775776380.220152:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776380.220155:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710410240, portal 10 00000100:00000001:0.0:1775776380.220157:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776380.220159:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684367616 : -123823025184000 : ffff8f623491a300) 00000100:00000040:0.0:1775776380.220164:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491a300 x1862022710410240/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776380.220171:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776380.220173:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776380.220176:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc07f8. 00000100:00000200:0.0:1775776380.220179:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710410240, offset 0 00000400:00000200:0.0:1775776380.220183:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776380.220187:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776380.220195:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776380.220198:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776380.220200:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776380.220202:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776380.220205:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776380.220208:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453757:453757:254:4294967295] 192.168.201.115@tcp LPNI seq info [453682:453682:6:4294967295] 00000400:00000200:0.0:1775776380.220212:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776380.220218:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776380.220222:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776380.220226:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c100. 00000800:00000200:0.0:1775776380.220230:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776380.220233:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775776380.220236:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776380.220239:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776380.220242:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776380.220245:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307935244 00000100:00000001:0.0:1775776380.220248:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307939078 00000100:00000001:0.0:1775776380.220250:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307935244 00000100:00000001:0.0:1775776380.220252:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307939078 00000100:00000001:0.0:1775776380.220255:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776380.220257:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776380.220259:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776380.220261:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776380.220264:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776380.220265:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1775776380.220267:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776380.220268:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776380.220270:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776380.220272:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776380.347251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776380.347268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776380.347276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776380.347280:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776380.347283:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776380.347287:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776380.347290:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776380.347293:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776380.347296:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776380.347299:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776380.347303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776380.347304:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776380.347306:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776380.347307:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776380.347309:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776380.347310:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776380.347312:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776380.347314:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776380.347374:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776380.347378:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776380.347383:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776381.243281:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776381.243290:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776381.243292:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776381.243299:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776381.243308:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776381.243314:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776381.243317:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776381.243319:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776381.243322:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776381.371282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776381.371290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776381.371296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776381.371297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776381.371301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776381.371302:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776381.371359:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776381.371363:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776381.371366:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776382.267212:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776382.267220:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776382.267222:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776382.267228:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776382.267232:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776382.267236:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776382.267239:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776382.267241:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776382.267243:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776382.395264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776382.395272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776382.395279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776382.395281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776382.395285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776382.395286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776382.395351:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776382.395356:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776382.395360:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776383.291291:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776383.291300:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776383.291302:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776383.291309:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776383.291313:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776383.291319:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776383.291322:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776383.291324:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776383.291327:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1775776383.291441:0:2400:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:2.0:1775776383.291516:0:2400:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1775776383.291529:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776383.291533:0:2400:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:2.0:1775776383.291539:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776383.291543:0:2400:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:2.0:1775776383.291548:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775776383.291552:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cd00. 00000400:00000200:2.0:1775776383.291558:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775776383.291562:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775776383.291568:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0198 00000400:00000010:2.0:1775776383.291576:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0198. 00000100:00000001:2.0:1775776383.291582:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776383.291593:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6234919f80 x1862022710409984/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776383.291619:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776383.291622:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234919f80 x1862022710409984/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776383.291630:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776383.291632:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775776383.291645:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775776383.291647:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c600. 00000400:00000200:2.0:1775776383.291651:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775776383.291656:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775776383.291659:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0cc0 00000400:00000010:2.0:1775776383.291661:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0cc0. 00000100:00000001:2.0:1775776383.291664:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1775776383.291669:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:2.0:1775776383.291669:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6234919c00 x1862022710410112/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776383.291674:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776383.291679:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1775776383.291681:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776383.291686:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234919c00 x1862022710410112/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776383.291690:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776380/real 1775776383] req@ffff8f6234919f80 x1862022710409984/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776383.291700:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776383.291704:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775776383.291712:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000400:00000001:0.0:1775776383.291714:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000010:2.0:1775776383.291714:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c100. 00000400:00000001:0.0:1775776383.291717:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1775776383.291718:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000010:0.0:1775776383.291723:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:2.0:1775776383.291724:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1775776383.291730:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776383.291733:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776383.291737:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:2.0:1775776383.291739:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc07f8 00000100:00000200:0.0:1775776383.291744:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234919f80 x1862022710409984/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1775776383.291744:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc07f8. 00000100:00000001:2.0:1775776383.291750:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776383.291759:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f623491a300 x1862022710410240/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776383.291760:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234919f80 x1862022710409984/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776383.291787:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1775776383.291789:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1775776383.291791:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1775776383.291794:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491a300 x1862022710410240/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776383.291797:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776383.291801:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776383.291804:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776383.291806:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:0.0:1775776383.291809:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234919f80 x1862022710409984/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776383.291816:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776383.291822:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234919f80 x1862022710409984/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776383.291830:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776383.291833:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776383.291834:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776383.291836:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776383.291840:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776383.291842:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776383.291847:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776383.291853:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776383.291855:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776383.291859:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234919f80 x1862022710409984/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776383.291867:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234919f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710409984:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776383.291872:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776383.291874:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776383.291876:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234919f80 x1862022710409984/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776383.291882:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776383.291886:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776383.291891:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775776383.291896:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776383.291899:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776383.291901:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776383.291904:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776383.291907:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:0.0:1775776383.291913:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776383.291915:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776383.291917:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234919f80. 00000100:00000001:0.0:1775776383.291922:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776383.291924:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776383.291927:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776383.291929:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776383.291930:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776383.291931:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776383.291935:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776380/real 1775776383] req@ffff8f623491a300 x1862022710410240/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776383.291949:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776383.291950:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776383.291953:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d68. 00000400:00000200:0.0:1775776383.291955:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776383.291957:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776383.291959:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776383.291962:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491a300 x1862022710410240/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776383.291969:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491a300 x1862022710410240/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776383.291975:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776383.291976:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776383.291978:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776383.291979:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776383.291983:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491a300 x1862022710410240/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776383.291989:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776383.291993:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491a300 x1862022710410240/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776383.291999:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776383.292001:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776383.292003:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776383.292004:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776383.292006:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776383.292007:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776383.292010:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776383.292013:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776383.292016:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776383.292021:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491a300 x1862022710410240/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776383.292028:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710410240:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776383.292032:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776383.292035:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776380/real 1775776383] req@ffff8f6234919c00 x1862022710410112/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776383.292043:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776383.292044:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776383.292046:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776383.292049:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776383.292050:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776383.292052:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776383.292055:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234919c00 x1862022710410112/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776383.292063:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234919c00 x1862022710410112/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776383.292069:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776383.292070:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776383.292072:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776383.292073:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776383.292076:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234919c00 x1862022710410112/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776383.292081:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776383.292084:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234919c00 x1862022710410112/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776383.292089:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776383.292091:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776383.292093:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776383.292094:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776383.292095:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776383.292102:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776383.292105:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776383.292108:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776383.292111:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776383.292114:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234919c00 x1862022710410112/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776383.292121:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234919c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710410112:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776383.292126:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776383.292127:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776383.292130:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491a300 x1862022710410240/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776383.292167:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776383.292170:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776383.292172:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776383.292175:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776383.292177:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776383.292179:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776383.292181:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776383.292183:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4c000. 02000000:00000001:0.0:1775776383.292187:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776383.292188:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776383.292190:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491a300. 00000100:00000001:0.0:1775776383.292193:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776383.292194:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776383.292196:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776383.292201:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234919c00 x1862022710410112/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776383.292208:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776383.292210:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776383.292212:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775776383.292215:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776383.292216:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776383.292217:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776383.292219:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776383.292221:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:0.0:1775776383.292222:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776383.292224:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776383.292225:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234919c00. 00000100:00000001:0.0:1775776383.292227:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776383.292228:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776383.292230:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776383.292241:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776383.292243:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776383.292246:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776383.292247:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776383.419243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776383.419248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776383.419254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776383.419255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776383.419259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776383.419260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776383.419319:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776383.419322:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776383.419325:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776384.123263:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307939078, at 4307939136 00000100:00000040:0.0:1775776384.123282:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307939078, at 4307939136 00000100:00000010:1.0:1775776384.123287:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3a20. 00000100:00000010:0.0:1775776384.123291:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776384.315322:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776384.315332:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776384.315334:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776384.315341:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776384.315345:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776384.315350:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776384.315353:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776384.315356:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776384.315358:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776384.443279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776384.443287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776384.443295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776384.443296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776384.443307:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776384.443308:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776384.443369:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776384.443374:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776384.443378:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776385.339269:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776385.339279:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776385.339282:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776385.339285:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776385.339289:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776385.339295:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13267 00000400:00000200:3.0:1775776385.339307:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776385.339313:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000400:00000200:3.0:1775776385.339315:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:0.0:1775776385.339319:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1775776385.339319:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775776385.339323:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776385.339324:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00100000:0.0:1775776385.339328:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776385.339328:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776385.339333:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776385.339334:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776385.339336:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:0.0:1775776385.339338:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775776385.339341:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:0.0:1775776385.339342:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1775776385.339344:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000100:00000001:0.0:1775776385.339346:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1775776385.339348:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775776385.339351:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776385.339352:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776385.339356:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776385.339359:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776385.339363:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776385.339364:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776385.339366:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776385.339368:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776385.339371:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776385.339374:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776385.339377:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776385.339381:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776385.339385:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776385.339387:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776385.339390:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776385.339391:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776385.339407:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775776385.339411:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776385.339418:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776385.339420:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776385.339422:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776385.339425:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776385.339429:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776385.339447:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000100:00000001:3.0:1775776385.339454:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776385.339463:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710410368/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776385.339472:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776385.339476:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776385.339480:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710410368/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776385.339505:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776385.339508:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776385.339511:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776385.339513:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776385.339514:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776385.339516:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776385.339519:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13267 00000400:00000200:3.0:1775776385.339522:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776385.339525:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776385.339526:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776385.339529:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776385.339531:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776385.339533:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776385.339534:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776385.339536:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776385.339537:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776385.339539:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776385.339541:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776385.339543:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776385.339544:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776385.339547:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776385.339548:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776385.339550:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776385.339552:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776385.339554:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776385.339556:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776385.339558:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776385.339559:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776385.339564:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776385.339566:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776385.339568:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776385.339572:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776385.339575:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776385.339576:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776385.339579:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775776385.339582:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776385.339584:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776385.339585:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776385.339587:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776385.339588:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776385.339589:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776385.339593:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000100:00000001:3.0:1775776385.339595:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776385.339600:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710410496/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776385.339608:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776385.339610:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776385.339613:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710410496/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776385.339621:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776385.339623:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776385.339626:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776385.339627:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776385.339628:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776385.339631:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776385.339633:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13267 00000400:00000200:3.0:1775776385.339636:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776385.339639:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776385.339641:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776385.339643:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776385.339645:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776385.339646:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776385.339648:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776385.339650:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776385.339651:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776385.339652:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776385.339654:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776385.339656:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776385.339658:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776385.339660:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776385.339662:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776385.339663:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776385.339665:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776385.339667:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776385.339670:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776385.339672:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776385.339673:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776385.339676:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776385.339677:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776385.339679:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776385.339681:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776385.339683:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776385.339684:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776385.339687:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000020:00000040:3.0:1775776385.339688:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776385.339691:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776385.339692:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776385.339693:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776385.339694:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776385.339696:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776385.339699:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000100:00000001:3.0:1775776385.339702:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776385.339706:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710410624/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776385.339712:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776385.339714:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776385.339718:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710410624/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776385.339724:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776385.339727:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776385.339730:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776385.339732:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776385.339735:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776385.339740:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13278) 00000100:00000001:0.0:1775776385.339802:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776385.339805:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776385.339807:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776385.339813:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71180 x1862022710410368/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.339820:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776385.339821:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776385.339824:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776385.339827:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776385.339829:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776385.339832:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710410368:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776385.339836:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776385.339840:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776385.339842:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776385.339844:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776385.339849:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775776385.339851:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776385.339857:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776385.339861:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776385.339867:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776385.339873:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710410368, portal 25 00000100:00000001:0.0:1775776385.339876:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776385.339877:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849008512 : -123823860543104 : ffff8f6202c71180) 00000100:00000040:0.0:1775776385.339882:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71180 x1862022710410368/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.339888:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776385.339891:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776385.339894:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776385.339896:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710410368, offset 0 00000400:00000200:0.0:1775776385.339904:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776385.339911:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776385.339917:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776385.339921:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776385.339925:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776385.339929:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776385.339931:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776385.339936:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453758:453758:256:4294967295] 192.168.201.115@tcp LPNI seq info [453683:453683:8:4294967295] 00000400:00000200:0.0:1775776385.339940:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776385.339948:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776385.339954:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776385.339958:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776385.339965:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776385.339969:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775776385.339973:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776385.339977:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775776385.339980:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776385.339982:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776385.339988:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776385.339993:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776385.339998:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776385.340000:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776385.340004:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776385.340008:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71180 x1862022710410368/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340016:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776385.340019:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71180 x1862022710410368/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776410 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340026:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776385.340027:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776385.340033:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776385.340036:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776385.340039:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307944198 00000100:00000010:0.0:1775776385.340043:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776385.340049:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776385.340055:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307944198 00000100:00000010:0.0:1775776385.340059:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c38a0. 00000100:00000001:0.0:1775776385.340062:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776385.340066:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776385.340070:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776385.340072:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776385.340076:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776385/real 1775776385] req@ffff8f6202c71180 x1862022710410368/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776385.340086:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776385.340087:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776385.340089:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776385.340092:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776385.340093:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776385.340095:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776385.340099:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71180 x1862022710410368/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776385.340107:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71180 x1862022710410368/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340113:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776385.340115:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776385.340117:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776385.340119:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776385.340123:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71180 x1862022710410368/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340131:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776385.340164:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71180 x1862022710410368/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340172:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776385.340175:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776385.340177:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776385.340178:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776385.340181:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776385.340183:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776385.340186:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776385.340190:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776385.340191:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776385.340197:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71180 x1862022710410368/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776385.340207:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710410368:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776385.340212:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776385.340216:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70700 x1862022710410496/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340222:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776385.340224:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776385.340225:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776385.340227:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776385.340229:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776385.340231:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710410496:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776385.340234:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776385.340235:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776385.340236:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776385.340238:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776385.340241:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776385.340243:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776385.340247:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776385.340249:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776385.340253:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776385.340255:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710410496, portal 10 00000100:00000001:0.0:1775776385.340257:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776385.340258:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849005824 : -123823860545792 : ffff8f6202c70700) 00000100:00000040:0.0:1775776385.340263:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70700 x1862022710410496/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340271:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776385.340273:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776385.340275:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776385.340277:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710410496, offset 0 00000400:00000200:0.0:1775776385.340281:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776385.340285:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776385.340289:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776385.340292:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776385.340294:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776385.340296:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776385.340298:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776385.340301:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453759:453759:256:4294967295] 192.168.201.115@tcp LPNI seq info [453684:453684:8:4294967295] 00000400:00000200:0.0:1775776385.340305:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776385.340311:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776385.340315:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776385.340318:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776385.340322:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776385.340324:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775776385.340327:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776385.340330:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775776385.340334:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776385.340336:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776385.340338:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776385.340342:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776385.340345:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776385.340347:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776385.340349:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776385.340353:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70700 x1862022710410496/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340361:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776385.340363:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70700 x1862022710410496/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776410 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340374:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776385.340375:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776385.340383:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776385.340386:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776385.340390:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776385.340392:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307940364 00000100:00000001:0.0:1775776385.340395:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307944198 00000100:00000001:0.0:1775776385.340397:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307940364 00000100:00000001:0.0:1775776385.340400:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307944198 00000100:00000001:0.0:1775776385.340402:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776385.340404:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776385.340406:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776385.340410:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776385/real 1775776385] req@ffff8f6202c70700 x1862022710410496/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776385.340418:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776385.340419:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776385.340421:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776385.340424:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776385.340425:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776385.340428:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776385.340441:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70700 x1862022710410496/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776385.340449:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70700 x1862022710410496/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340456:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776385.340458:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776385.340459:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776385.340461:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776385.340464:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70700 x1862022710410496/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340470:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776385.340474:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70700 x1862022710410496/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340479:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776385.340481:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776385.340482:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776385.340484:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776385.340485:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776385.340490:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776385.340492:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776385.340495:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776385.340497:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776385.340500:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70700 x1862022710410496/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776385.340506:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710410496:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776385.340509:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776385.340514:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70a80 x1862022710410624/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340522:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776385.340523:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776385.340527:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776385.340528:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776385.340530:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776385.340532:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710410624:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776385.340535:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776385.340540:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776385.340543:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776385.340545:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776385.340549:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775776385.340550:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776385.340554:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776385.340556:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776385.340558:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776385.340561:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710410624, portal 10 00000100:00000001:0.0:1775776385.340562:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776385.340563:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849006720 : -123823860544896 : ffff8f6202c70a80) 00000100:00000040:0.0:1775776385.340569:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70a80 x1862022710410624/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340575:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776385.340577:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776385.340579:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776385.340582:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710410624, offset 0 00000400:00000200:0.0:1775776385.340585:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776385.340589:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776385.340592:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776385.340595:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776385.340597:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776385.340599:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776385.340601:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776385.340604:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453760:453760:256:4294967295] 192.168.201.115@tcp LPNI seq info [453685:453685:8:4294967295] 00000400:00000200:0.0:1775776385.340607:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776385.340613:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776385.340617:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776385.340620:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776385.340624:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776385.340626:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775776385.340630:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776385.340633:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775776385.340636:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776385.340639:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776385.340642:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776385.340645:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776385.340653:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776385.340655:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776385.340657:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776385.340660:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70a80 x1862022710410624/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340668:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776385.340671:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70a80 x1862022710410624/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776410 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340677:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776385.340679:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776385.340682:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776385.340685:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776385.340688:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776385.340690:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307940364 00000100:00000001:0.0:1775776385.340692:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307944198 00000100:00000001:0.0:1775776385.340694:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307940364 00000100:00000001:0.0:1775776385.340697:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307944198 00000100:00000001:0.0:1775776385.340700:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776385.340702:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776385.340705:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776385.340708:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776385/real 1775776385] req@ffff8f6202c70a80 x1862022710410624/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776385.340716:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776385.340718:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776385.340719:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776385.340721:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776385.340723:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776385.340726:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776385.340728:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70a80 x1862022710410624/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776385.340736:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70a80 x1862022710410624/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340742:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776385.340743:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776385.340745:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776385.340746:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776385.340749:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70a80 x1862022710410624/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340756:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776385.340759:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70a80 x1862022710410624/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340766:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776385.340768:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776385.340770:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776385.340771:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776385.340773:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776385.340776:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776385.340778:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776385.340782:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776385.340784:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776385.340788:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70a80 x1862022710410624/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776385.340796:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710410624:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776385.340799:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776385.340801:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776385.340804:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71180 x1862022710410368/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340810:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776385.340812:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776385.340814:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775776385.340817:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776385.340820:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776385.340821:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776385.340824:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776385.340826:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36c00. 02000000:00000001:0.0:1775776385.340830:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776385.340832:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776385.340834:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71180. 00000100:00000001:0.0:1775776385.340838:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776385.340839:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776385.340841:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776385.340844:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70700 x1862022710410496/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340850:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776385.340851:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776385.340853:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776385.340855:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776385.340856:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776385.340857:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776385.340859:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776385.340860:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35c00. 02000000:00000001:0.0:1775776385.340862:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776385.340863:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776385.340864:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70700. 00000100:00000001:0.0:1775776385.340866:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776385.340867:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776385.340868:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776385.340870:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70a80 x1862022710410624/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776410 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776385.340876:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776385.340878:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776385.340880:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775776385.340881:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776385.340883:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776385.340883:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776385.340885:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776385.340886:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34000. 02000000:00000001:0.0:1775776385.340888:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776385.340889:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776385.340891:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70a80. 00000100:00000001:0.0:1775776385.340892:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776385.340893:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776385.340895:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776385.340902:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776385.340904:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776385.340906:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776385.340907:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776385.467224:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776385.467232:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776385.467234:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776385.467237:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776385.467240:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776385.467241:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776385.467243:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776385.467244:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776385.467247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776385.467248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776385.467251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776385.467251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776385.467308:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776385.467311:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776385.467313:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776386.363289:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776386.363295:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776386.363296:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776386.363300:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776386.363303:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776386.363306:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776386.363307:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776386.363309:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776386.363311:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776386.491255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776386.491260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776386.491265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776386.491267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776386.491270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776386.491271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776386.491330:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776386.491332:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776386.491334:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776387.387226:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776387.387231:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776387.387233:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776387.387238:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776387.387242:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776387.387245:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776387.387247:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776387.387250:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776387.387252:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776387.515195:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776387.515199:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776387.515203:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776387.515204:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776387.515206:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776387.515207:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776387.515277:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776387.515281:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776387.515285:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776388.411277:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776388.411284:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776388.411286:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776388.411292:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776388.411295:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776388.411299:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776388.411301:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776388.411304:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776388.411307:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776388.539223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776388.539232:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776388.539238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776388.539240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776388.539244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776388.539246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776388.539295:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776388.539299:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776388.539302:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776389.243255:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307944198, at 4307944256 00000100:00000040:0.0:1775776389.243269:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307944198, at 4307944256 00000100:00000010:1.0:1775776389.243271:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c38a0. 00000100:00000010:0.0:1775776389.243283:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776389.435203:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776389.435207:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776389.435209:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776389.435213:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776389.435216:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776389.435219:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776389.435220:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776389.435222:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776389.435224:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776389.563267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776389.563272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776389.563277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776389.563278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776389.563281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776389.563281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776389.563333:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776389.563337:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776389.563340:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776390.459234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776390.459239:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776390.459241:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776390.459245:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776390.459251:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776390.459254:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776390.459256:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776390.459258:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776390.459260:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776390.459261:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776390.459267:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776390.459269:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776390.459271:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776390.459274:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776390.459281:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13273 00000400:00000200:1.0:1775776390.459291:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776390.459295:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776390.459297:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776390.459300:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776390.459302:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776390.459304:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776390.459305:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:1.0:1775776390.459308:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776390.459309:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:1.0:1775776390.459311:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776390.459313:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:1.0:1775776390.459315:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:1.0:1775776390.459316:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:1.0:1775776390.459318:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776390.459319:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776390.459320:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776390.459321:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:1.0:1775776390.459322:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:1.0:1775776390.459324:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776390.459325:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776390.459327:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776390.459329:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776390.459331:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776390.459333:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776390.459334:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776390.459348:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623491a680. 00000020:00000040:1.0:1775776390.459351:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:1.0:1775776390.459353:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776390.459354:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776390.459356:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776390.459358:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776390.459362:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776390.459368:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4fc00. 00000100:00000001:1.0:1775776390.459373:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776390.459379:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623491a680 x1862022710410752/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776390.459386:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776390.459390:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776390.459392:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029bc1460] to pc [ptlrpcd_rcv+-1] req@ffff8f623491a680 x1862022710410752/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776390.459406:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776390.459408:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776390.459411:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776390.459412:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776390.459413:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776390.459415:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776390.459417:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13273 00000400:00000200:1.0:1775776390.459420:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776390.459422:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776390.459424:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775776390.459425:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776390.459428:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1775776390.459428:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776390.459430:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776390.459431:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776390.459433:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776390.459435:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:0.0:1775776390.459438:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491a680 x1862022710410752/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775776390.459438:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776390.459439:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:1.0:1775776390.459441:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776390.459444:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:1.0:1775776390.459448:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:1.0:1775776390.459452:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775776390.459453:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776390.459455:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776390.459456:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776390.459460:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775776390.459461:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1775776390.459461:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776390.459464:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:0.0:1775776390.459465:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1775776390.459467:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:0.0:1775776390.459469:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776390.459470:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776390.459473:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776390.459474:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710410752:192.168.201.115@tcp:250:kworker.0 02000000:00000001:1.0:1775776390.459475:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775776390.459479:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776390.459480:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776390.459482:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776390.459483:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776390.459484:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776390.459485:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775776390.459487:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:1.0:1775776390.459489:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776390.459492:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776390.459493:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000001:1.0:1775776390.459493:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776390.459497:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776390.459500:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6234919c00. 00000020:00000040:1.0:1775776390.459502:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000400:00000200:0.0:1775776390.459505:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:1.0:1775776390.459506:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776390.459508:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776390.459509:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000100:00000001:1.0:1775776390.459509:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776390.459512:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776390.459514:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776390.459516:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 02000000:00000010:1.0:1775776390.459517:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4f400. 00000100:00000200:0.0:1775776390.459522:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710410752, portal 25 00000100:00000001:0.0:1775776390.459526:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776390.459527:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776390.459528:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684368512 : -123823025183104 : ffff8f623491a680) 00000100:00100000:1.0:1775776390.459532:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6234919c00 x1862022710410880/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1775776390.459536:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491a680 x1862022710410752/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776390.459545:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776390.459548:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776390.459550:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776390.459553:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000718661bc] to pc [ptlrpcd_rcv+-1] req@ffff8f6234919c00 x1862022710410880/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776390.459554:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776390.459558:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776390.459562:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710410752, offset 0 00000100:00000001:1.0:1775776390.459566:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776390.459568:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775776390.459570:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00080000:1.0:1775776390.459573:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776390.459575:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776390.459575:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776390.459577:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:0.0:1775776390.459579:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00080000:1.0:1775776390.459582:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13273 00000400:00000200:0.0:1775776390.459587:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776390.459590:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776390.459593:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776390.459596:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776390.459597:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776390.459601:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453761:453761:256:4294967295] 192.168.201.115@tcp LPNI seq info [453686:453686:8:4294967295] 00000400:00000200:0.0:1775776390.459605:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776390.459614:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000400:00000200:1.0:1775776390.459620:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000200:0.0:1775776390.459622:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776390.459625:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776390.459627:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000800:00000010:0.0:1775776390.459629:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000100:00000040:1.0:1775776390.459629:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776390.459630:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776390.459634:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776390.459636:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:1.0:1775776390.459637:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776390.459639:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000800:00000200:0.0:1775776390.459642:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:1.0:1775776390.459642:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776390.459644:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:1.0:1775776390.459647:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:0.0:1775776390.459648:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1775776390.459649:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:1.0:1775776390.459653:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776390.459655:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776390.459656:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776390.459659:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000200:0.1:1775776390.459660:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000040:1.0:1775776390.459661:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:1.0:1775776390.459666:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000800:00000200:0.1:1775776390.459667:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000100:00000001:1.0:1775776390.459668:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776390.459670:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776390.459672:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776390.459674:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776390.459676:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000100:0.0:1775776390.459678:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000008:00100000:1.0:1775776390.459679:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776390.459682:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775776390.459683:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000100:00000001:1.0:1775776390.459683:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776390.459688:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623491a300. 00000020:00000040:1.0:1775776390.459690:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:0.0:1775776390.459692:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:1.0:1775776390.459692:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776390.459693:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776390.459696:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776390.459698:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776390.459699:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776390.459703:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 02000000:00000010:1.0:1775776390.459703:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4ec00. 00000100:00000001:1.0:1775776390.459706:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776390.459709:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000100:00100000:1.0:1775776390.459711:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623491a300 x1862022710411008/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1775776390.459712:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776390.459718:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776390.459724:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491a680 x1862022710410752/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776390.459728:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776390.459731:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776390.459737:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bfa067b] to pc [ptlrpcd_rcv+-1] req@ffff8f623491a300 x1862022710411008/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776390.459741:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776390.459743:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776390.459745:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776390.459748:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776390.459751:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491a680 x1862022710410752/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776415 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776390.459751:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1775776390.459759:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1775776390.459764:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776390.459764:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13283) 00000100:00000001:0.0:1775776390.459766:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776390.459771:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776390.459775:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776390.459777:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307949318 00000100:00000010:0.0:1775776390.459781:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776390.459786:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776390.459790:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307949318 00000100:00000010:0.0:1775776390.459792:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c37e0. 00000100:00000001:0.0:1775776390.459794:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776390.459796:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776390.459798:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776390.459799:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776390.459801:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776390/real 1775776390] req@ffff8f623491a680 x1862022710410752/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776390.459812:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776390.459813:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776390.459814:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776390.459816:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776390.459816:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776390.459818:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776390.459820:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491a680 x1862022710410752/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776390.459825:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491a680 x1862022710410752/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776390.459828:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776390.459829:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776390.459830:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776390.459832:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776390.459834:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491a680 x1862022710410752/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776390.459837:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776390.459840:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491a680 x1862022710410752/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776390.459844:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776390.459846:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776390.459846:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776390.459847:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776390.459849:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776390.459850:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776390.459851:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776390.459854:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776390.459855:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776390.459857:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491a680 x1862022710410752/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776390.459862:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710410752:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776390.459865:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776390.459866:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776390.459867:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491a680 x1862022710410752/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776390.459871:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776390.459872:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776390.459874:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775776390.459875:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776390.459876:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776390.459877:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776390.459878:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776390.459880:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4fc00. 02000000:00000001:0.0:1775776390.459882:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776390.459883:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776390.459884:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491a680. 00000100:00000001:0.0:1775776390.459886:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776390.459886:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776390.459888:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776390.459892:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776390.459893:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776390.459895:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776390.459896:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776390.459896:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776390.459898:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234919c00 x1862022710410880/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776390.459902:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776390.459903:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776390.459904:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776390.459906:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776390.459907:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776390.459908:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234919c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710410880:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776390.459910:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776390.459912:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776390.459913:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776390.459914:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776390.459915:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775776390.459916:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776390.459919:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776390.459920:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776390.459922:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776390.459924:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710410880, portal 10 00000100:00000001:0.0:1775776390.459925:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776390.459926:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684365824 : -123823025185792 : ffff8f6234919c00) 00000100:00000040:0.0:1775776390.459930:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234919c00 x1862022710410880/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776390.459934:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776390.459935:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776390.459936:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776390.459937:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710410880, offset 0 00000400:00000200:0.0:1775776390.459940:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776390.459942:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776390.459944:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776390.459946:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776390.459947:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776390.459949:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776390.459950:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776390.459952:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453762:453762:256:4294967295] 192.168.201.115@tcp LPNI seq info [453687:453687:8:4294967295] 00000400:00000200:0.0:1775776390.459955:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776390.459959:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776390.459961:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776390.459963:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776390.459965:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776390.459967:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775776390.459970:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776390.459971:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1775776390.459973:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776390.459975:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776390.459976:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776390.459978:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776390.459980:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776390.459981:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776390.459982:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776390.459984:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234919c00 x1862022710410880/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776390.459988:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776390.459990:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234919c00 x1862022710410880/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776415 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776390.459993:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776390.459994:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776390.459996:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776390.459998:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776390.460000:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776390.460001:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307945483 00000100:00000001:0.0:1775776390.460003:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307949318 00000100:00000001:0.0:1775776390.460004:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307945483 00000100:00000001:0.0:1775776390.460005:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307949318 00000100:00000001:0.0:1775776390.460006:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776390.460007:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776390.460008:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776390.460010:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776390/real 1775776390] req@ffff8f6234919c00 x1862022710410880/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776390.460014:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776390.460015:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776390.460016:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776390.460018:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776390.460019:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776390.460020:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776390.460021:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234919c00 x1862022710410880/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776390.460026:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234919c00 x1862022710410880/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776390.460029:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776390.460030:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776390.460031:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776390.460031:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776390.460033:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234919c00 x1862022710410880/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776390.460039:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776390.460041:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234919c00 x1862022710410880/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776390.460045:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776390.460047:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776390.460048:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776390.460049:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776390.460051:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776390.460055:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776390.460057:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776390.460060:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776390.460062:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776390.460066:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234919c00 x1862022710410880/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776390.460073:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234919c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710410880:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776390.460077:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776390.460080:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491a300 x1862022710411008/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776390.460086:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776390.460088:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776390.460090:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776390.460092:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776390.460094:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776390.460096:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710411008:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776390.460099:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776390.460101:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776390.460103:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776390.460105:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776390.460108:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776390.460109:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776390.460111:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776390.460113:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776390.460115:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776390.460117:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710411008, portal 10 00000100:00000001:0.0:1775776390.460118:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776390.460119:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684367616 : -123823025184000 : ffff8f623491a300) 00000100:00000040:0.0:1775776390.460123:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491a300 x1862022710411008/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776390.460127:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776390.460128:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776390.460130:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776390.460131:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710411008, offset 0 00000400:00000200:0.0:1775776390.460148:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776390.460150:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776390.460152:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776390.460154:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776390.460156:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776390.460158:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776390.460159:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776390.460161:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453763:453763:256:4294967295] 192.168.201.115@tcp LPNI seq info [453688:453688:8:4294967295] 00000400:00000200:0.0:1775776390.460163:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776390.460167:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776390.460169:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776390.460171:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776390.460174:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776390.460175:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775776390.460178:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776390.460179:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1775776390.460181:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776390.460182:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776390.460184:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776390.460186:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776390.460188:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776390.460189:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776390.460190:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776390.460192:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491a300 x1862022710411008/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776390.460197:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776390.460198:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491a300 x1862022710411008/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776415 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776390.460202:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776390.460203:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776390.460205:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776390.460207:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776390.460208:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776390.460209:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307945484 00000100:00000001:0.0:1775776390.460211:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307949318 00000100:00000001:0.0:1775776390.460212:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307945484 00000100:00000001:0.0:1775776390.460214:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307949318 00000100:00000001:0.0:1775776390.460215:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776390.460217:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776390.460218:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776390.460220:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776390/real 1775776390] req@ffff8f623491a300 x1862022710411008/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776390.460224:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776390.460225:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776390.460225:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776390.460227:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776390.460227:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776390.460228:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776390.460230:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491a300 x1862022710411008/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776390.460235:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491a300 x1862022710411008/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776390.460238:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776390.460239:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776390.460241:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776390.460242:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776390.460245:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491a300 x1862022710411008/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776390.460251:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776390.460254:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491a300 x1862022710411008/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776390.460260:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776390.460263:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776390.460264:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776390.460265:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776390.460267:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776390.460269:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776390.460272:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776390.460275:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776390.460277:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776390.460281:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491a300 x1862022710411008/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776390.460288:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710411008:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776390.460292:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776390.460294:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776390.460297:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234919c00 x1862022710410880/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776390.460303:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776390.460304:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776390.460306:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775776390.460309:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776390.460310:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776390.460312:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776390.460314:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776390.460315:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4f400. 02000000:00000001:0.0:1775776390.460318:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776390.460319:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776390.460320:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234919c00. 00000100:00000001:0.0:1775776390.460323:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776390.460324:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776390.460326:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776390.460329:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491a300 x1862022710411008/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776415 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776390.460336:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776390.460337:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776390.460338:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776390.460354:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776390.460355:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776390.460356:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776390.460358:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776390.460359:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:0.0:1775776390.460360:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776390.460361:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776390.460362:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491a300. 00000100:00000001:0.0:1775776390.460363:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776390.460364:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776390.460365:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776390.460368:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776390.460368:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776390.460370:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776390.460371:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776390.587240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776390.587244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776390.587247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776390.587250:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776390.587251:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776390.587253:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776390.587255:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776390.587257:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776390.587259:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776390.587261:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776390.587265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776390.587270:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776390.587271:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776390.587272:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776390.587274:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776390.587275:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776390.587277:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776390.587278:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776390.587369:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776390.587373:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776390.587376:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776391.483222:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776391.483227:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776391.483229:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776391.483234:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776391.483236:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776391.483239:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776391.483241:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776391.483242:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776391.483244:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776391.611237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776391.611241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776391.611245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776391.611246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776391.611248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776391.611249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776391.611318:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776391.611331:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776391.611335:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776392.507280:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776392.507287:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776392.507288:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776392.507294:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776392.507297:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776392.507302:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776392.507304:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776392.507318:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776392.507321:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776392.635211:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776392.635216:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776392.635221:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776392.635222:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776392.635225:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776392.635225:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776392.635264:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776392.635268:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776392.635271:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776393.531243:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776393.531250:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776393.531251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776393.531257:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776393.531260:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776393.531265:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776393.531267:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776393.531270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776393.531272:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776393.659238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776393.659241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776393.659245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776393.659246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776393.659248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776393.659249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776393.659316:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776393.659319:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776393.659322:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776394.363233:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307949318, at 4307949376 00000100:00000010:1.0:1775776394.363245:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c37e0. 00000100:00000040:0.0:1775776394.363296:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307949318, at 4307949376 00000100:00000010:0.0:1775776394.363309:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776394.555281:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776394.555290:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776394.555291:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776394.555298:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776394.555301:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776394.555305:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776394.555307:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776394.555309:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776394.555311:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776394.683262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776394.683290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776394.683296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776394.683297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776394.683300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776394.683301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776394.683359:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776394.683362:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776394.683365:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776395.579245:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776395.579247:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:1.0:1775776395.579264:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776395.579265:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776395.579266:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775776395.579268:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776395.579271:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776395.579275:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00100000:0.0:1775776395.579280:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:1.0:1775776395.579283:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13278 00000100:00000001:0.0:1775776395.579286:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1775776395.579293:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776395.579295:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776395.579299:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:1.0:1775776395.579301:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:0.0:1775776395.579303:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775776395.579306:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775776395.579308:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776395.579313:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776395.579317:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776395.579319:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776395.579322:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:1.0:1775776395.579326:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776395.579328:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:1.0:1775776395.579331:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776395.579335:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:1.0:1775776395.579338:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:1.0:1775776395.579341:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:1.0:1775776395.579344:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776395.579346:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776395.579348:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776395.579349:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:1.0:1775776395.579352:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:1.0:1775776395.579354:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776395.579356:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776395.579358:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776395.579361:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776395.579363:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776395.579364:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776395.579366:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776395.579375:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6234919f80. 00000020:00000040:1.0:1775776395.579378:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:1.0:1775776395.579381:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776395.579383:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776395.579385:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776395.579388:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776395.579392:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776395.579399:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4d400. 00000100:00000001:1.0:1775776395.579406:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776395.579414:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6234919f80 x1862022710411136/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776395.579422:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776395.579426:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776395.579430:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0867d60] to pc [ptlrpcd_rcv+-1] req@ffff8f6234919f80 x1862022710411136/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776395.579447:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776395.579449:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776395.579451:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776395.579452:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776395.579453:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776395.579455:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776395.579457:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13278 00000400:00000200:1.0:1775776395.579460:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776395.579462:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776395.579464:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776395.579466:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776395.579467:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776395.579469:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776395.579471:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:1.0:1775776395.579472:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776395.579473:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:1.0:1775776395.579474:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776395.579476:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:0.0:1775776395.579477:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1775776395.579477:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:1.0:1775776395.579480:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775776395.579482:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1775776395.579482:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776395.579485:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776395.579486:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776395.579487:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776395.579491:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:1.0:1775776395.579494:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000040:0.0:1775776395.579496:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234919f80 x1862022710411136/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775776395.579499:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776395.579504:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776395.579508:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775776395.579512:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776395.579514:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776395.579516:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776395.579518:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776395.579520:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776395.579522:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776395.579523:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775776395.579527:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1775776395.579530:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775776395.579534:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234919f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710411136:192.168.201.115@tcp:250:kworker.0 00000008:00000001:1.0:1775776395.579535:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776395.579539:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776395.579542:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000010:1.0:1775776395.579543:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6234918380. 00000020:00000040:1.0:1775776395.579546:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776395.579547:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776395.579548:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776395.579550:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776395.579551:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776395.579551:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776395.579554:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776395.579556:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1775776395.579556:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776395.579560:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4f000. 00000100:00000001:1.0:1775776395.579563:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776395.579564:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776395.579568:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776395.579568:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6234918380 x1862022710411264/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775776395.579578:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776395.579581:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:0.0:1775776395.579583:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000100:00000001:1.0:1775776395.579584:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776395.579591:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072d9a890] to pc [ptlrpcd_rcv+-1] req@ffff8f6234918380 x1862022710411264/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1775776395.579593:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776395.579603:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710411136, portal 25 00000100:00000001:1.0:1775776395.579603:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776395.579606:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776395.579607:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776395.579610:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684366720 : -123823025184896 : ffff8f6234919f80) 00000100:00080000:1.0:1775776395.579612:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776395.579615:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776395.579617:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:0.0:1775776395.579619:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234919f80 x1862022710411136/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776395.579622:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776395.579628:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13278 00000100:00000001:0.0:1775776395.579632:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:1.0:1775776395.579634:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:0.0:1775776395.579638:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776395.579640:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000400:00000010:0.0:1775776395.579642:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00080000:1.0:1775776395.579642:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776395.579647:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776395.579648:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776395.579650:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776395.579651:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:1.0:1775776395.579652:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776395.579652:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:1.0:1775776395.579653:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776395.579654:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:1.0:1775776395.579656:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000200:0.0:1775776395.579657:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710411136, offset 0 00000020:00000001:1.0:1775776395.579657:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:1.0:1775776395.579662:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776395.579664:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776395.579666:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776395.579669:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:0.0:1775776395.579670:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000020:00000040:1.0:1775776395.579671:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:1.0:1775776395.579675:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776395.579676:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776395.579677:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776395.579679:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775776395.579683:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:1.0:1775776395.579683:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776395.579685:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776395.579687:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776395.579688:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776395.579690:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776395.579693:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000010:1.0:1775776395.579693:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6234918e00. 00000020:00000040:1.0:1775776395.579697:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776395.579699:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1775776395.579701:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:1.0:1775776395.579702:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776395.579705:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776395.579706:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776395.579707:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776395.579709:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 02000000:00000010:1.0:1775776395.579713:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4ec00. 00000400:00000200:0.0:1775776395.579715:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1775776395.579717:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776395.579722:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00100000:1.0:1775776395.579722:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6234918e00 x1862022710411392/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775776395.579731:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453764:453764:256:4294967295] 192.168.201.115@tcp LPNI seq info [453689:453689:8:4294967295] 00000100:00000001:1.0:1775776395.579735:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776395.579739:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776395.579740:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:1.0:1775776395.579746:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000416b2640] to pc [ptlrpcd_rcv+-1] req@ffff8f6234918e00 x1862022710411392/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776395.579760:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:1.0:1775776395.579761:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776395.579766:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.0:1775776395.579772:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000040:1.0:1775776395.579772:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:1.0:1775776395.579776:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1775776395.579780:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000010:0.0:1775776395.579781:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000100:00000040:1.0:1775776395.579785:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13288) 00000800:00000200:0.0:1775776395.579793:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776395.579797:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775776395.579804:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776395.579808:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1775776395.579813:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776395.579815:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776395.579822:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776395.579826:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776395.579832:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776395.579834:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776395.579839:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776395.579844:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234919f80 x1862022710411136/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.579853:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776395.579856:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234919f80 x1862022710411136/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776420 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.579863:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776395.579865:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776395.579871:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776395.579875:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776395.579878:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307954438 00000100:00000010:0.0:1775776395.579883:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776395.579890:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776395.579897:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307954438 00000100:00000010:0.0:1775776395.579900:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c34e0. 00000100:00000001:0.0:1775776395.579904:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776395.579907:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776395.579910:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776395.579911:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776395.579915:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776395/real 1775776395] req@ffff8f6234919f80 x1862022710411136/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776395.579925:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776395.579927:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776395.579929:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776395.579932:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776395.579933:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776395.579936:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776395.579939:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234919f80 x1862022710411136/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776395.579948:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234919f80 x1862022710411136/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.579955:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776395.579956:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776395.579959:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776395.579961:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776395.579965:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234919f80 x1862022710411136/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.579972:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776395.579976:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234919f80 x1862022710411136/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.579983:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776395.579986:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776395.579987:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776395.579989:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776395.579991:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776395.579993:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776395.579996:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776395.580001:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776395.580003:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776395.580008:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234919f80 x1862022710411136/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776395.580016:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234919f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710411136:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776395.580022:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776395.580033:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776395.580037:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234919f80 x1862022710411136/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.580043:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776395.580046:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776395.580048:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776395.580052:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776395.580054:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776395.580055:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776395.580058:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776395.580060:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4d400. 02000000:00000001:0.0:1775776395.580064:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776395.580065:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776395.580067:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234919f80. 00000100:00000001:0.0:1775776395.580070:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776395.580072:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776395.580074:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776395.580089:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776395.580091:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776395.580093:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776395.580095:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776395.580096:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776395.580100:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918380 x1862022710411264/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.580107:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776395.580108:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776395.580110:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776395.580112:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776395.580114:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776395.580116:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710411264:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776395.580119:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776395.580121:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776395.580123:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776395.580125:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776395.580128:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776395.580130:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776395.580153:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776395.580157:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776395.580161:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776395.580164:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710411264, portal 10 00000100:00000001:0.0:1775776395.580166:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776395.580167:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684359552 : -123823025192064 : ffff8f6234918380) 00000100:00000040:0.0:1775776395.580174:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918380 x1862022710411264/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.580181:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776395.580183:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776395.580186:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776395.580188:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710411264, offset 0 00000400:00000200:0.0:1775776395.580192:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776395.580196:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776395.580200:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776395.580203:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776395.580206:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776395.580208:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776395.580210:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776395.580213:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453765:453765:256:4294967295] 192.168.201.115@tcp LPNI seq info [453690:453690:8:4294967295] 00000400:00000200:0.0:1775776395.580218:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776395.580224:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776395.580229:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776395.580232:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776395.580236:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776395.580238:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775776395.580242:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776395.580245:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1775776395.580248:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776395.580250:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776395.580272:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776395.580275:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776395.580278:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776395.580280:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776395.580282:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776395.580286:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918380 x1862022710411264/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.580293:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776395.580296:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918380 x1862022710411264/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776420 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.580302:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776395.580303:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776395.580306:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776395.580310:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776395.580313:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776395.580315:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307950604 00000100:00000001:0.0:1775776395.580317:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307954438 00000100:00000001:0.0:1775776395.580319:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307950604 00000100:00000001:0.0:1775776395.580322:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307954438 00000100:00000001:0.0:1775776395.580324:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776395.580326:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776395.580327:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776395.580331:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776395/real 1775776395] req@ffff8f6234918380 x1862022710411264/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776395.580338:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776395.580339:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776395.580341:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776395.580343:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776395.580345:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776395.580347:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776395.580350:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918380 x1862022710411264/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776395.580357:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918380 x1862022710411264/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.580364:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776395.580365:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776395.580366:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776395.580368:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776395.580371:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918380 x1862022710411264/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.580378:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776395.580382:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918380 x1862022710411264/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.580389:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776395.580391:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776395.580393:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776395.580394:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776395.580396:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776395.580400:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776395.580403:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776395.580406:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776395.580409:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776395.580413:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918380 x1862022710411264/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776395.580421:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710411264:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776395.580424:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776395.580428:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918e00 x1862022710411392/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.580440:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776395.580441:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776395.580443:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776395.580445:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776395.580447:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776395.580449:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710411392:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776395.580453:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776395.580454:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776395.580456:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776395.580458:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776395.580461:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775776395.580463:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776395.580466:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776395.580468:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776395.580471:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776395.580474:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710411392, portal 10 00000100:00000001:0.0:1775776395.580475:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776395.580477:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684362240 : -123823025189376 : ffff8f6234918e00) 00000100:00000040:0.0:1775776395.580482:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918e00 x1862022710411392/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.580488:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776395.580490:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776395.580492:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776395.580494:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710411392, offset 0 00000400:00000200:0.0:1775776395.580497:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776395.580501:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776395.580504:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776395.580507:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776395.580509:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776395.580511:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776395.580514:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776395.580517:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453766:453766:256:4294967295] 192.168.201.115@tcp LPNI seq info [453691:453691:8:4294967295] 00000400:00000200:0.0:1775776395.580521:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776395.580527:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776395.580531:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776395.580535:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776395.580538:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776395.580541:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775776395.580545:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776395.580547:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1775776395.580551:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776395.580553:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776395.580555:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776395.580558:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776395.580561:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776395.580563:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776395.580565:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776395.580568:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918e00 x1862022710411392/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.580576:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776395.580579:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918e00 x1862022710411392/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776420 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.580585:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776395.580586:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776395.580589:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776395.580592:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776395.580595:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776395.580597:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307950604 00000100:00000001:0.0:1775776395.580599:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307954438 00000100:00000001:0.0:1775776395.580601:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307950604 00000100:00000001:0.0:1775776395.580603:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307954438 00000100:00000001:0.0:1775776395.580605:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776395.580607:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776395.580609:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776395.580612:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776395/real 1775776395] req@ffff8f6234918e00 x1862022710411392/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776395.580619:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776395.580621:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776395.580622:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776395.580624:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776395.580625:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776395.580627:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776395.580630:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918e00 x1862022710411392/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776395.580638:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918e00 x1862022710411392/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.580643:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776395.580645:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776395.580646:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776395.580647:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776395.580650:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918e00 x1862022710411392/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.580656:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776395.580660:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918e00 x1862022710411392/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.580666:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776395.580669:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776395.580670:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776395.580672:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776395.580673:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776395.580675:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776395.580677:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776395.580681:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776395.580683:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776395.580687:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918e00 x1862022710411392/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776395.580695:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710411392:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776395.580698:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776395.580700:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776395.580702:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918380 x1862022710411264/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.580709:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776395.580711:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776395.580713:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776395.580715:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776395.580716:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776395.580717:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776395.580719:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776395.580721:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4f000. 02000000:00000001:0.0:1775776395.580723:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776395.580725:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776395.580726:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918380. 00000100:00000001:0.0:1775776395.580728:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776395.580730:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776395.580731:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776395.580734:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918e00 x1862022710411392/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776420 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776395.580740:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776395.580741:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776395.580743:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775776395.580745:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776395.580746:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776395.580747:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776395.580749:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776395.580751:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:0.0:1775776395.580753:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776395.580754:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776395.580756:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918e00. 00000100:00000001:0.0:1775776395.580758:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776395.580759:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776395.580760:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776395.580764:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776395.580765:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776395.580767:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776395.580768:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776395.707283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776395.707290:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776395.707292:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776395.707295:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776395.707300:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776395.707302:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776395.707304:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776395.707306:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776395.707309:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776395.707310:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776395.707313:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776395.707314:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776395.707343:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776395.707346:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776395.707349:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776396.603246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776396.603252:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776396.603254:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776396.603260:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776396.603265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776396.603269:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776396.603272:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776396.603276:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776396.603279:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776396.731265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776396.731270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776396.731276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776396.731277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776396.731279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776396.731280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776396.731341:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776396.731345:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776396.731349:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776397.627259:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776397.627268:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776397.627271:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776397.627278:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776397.627281:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776397.627286:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776397.627288:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776397.627290:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776397.627293:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776397.755284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776397.755290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776397.755295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776397.755296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776397.755298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776397.755299:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776397.755360:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776397.755364:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776397.755368:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776398.651259:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776398.651268:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776398.651271:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776398.651278:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776398.651282:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776398.651286:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776398.651289:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776398.651292:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776398.651295:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776398.779258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776398.779264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776398.779269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776398.779270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776398.779273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776398.779274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776398.779328:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776398.779331:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776398.779333:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776399.483237:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307954438, at 4307954496 00000100:00000010:1.0:1775776399.483248:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c34e0. 00000100:00000040:0.0:1775776399.483249:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307954438, at 4307954496 00000100:00000010:0.0:1775776399.483256:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776399.675251:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776399.675267:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776399.675270:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776399.675275:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776399.675277:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776399.675280:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776399.675282:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776399.675283:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776399.675285:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776399.803253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776399.803258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776399.803264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776399.803265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776399.803270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776399.803271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776399.803328:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776399.803331:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776399.803333:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776400.699250:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776400.699255:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776400.699258:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776400.699263:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776400.699267:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776400.699270:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776400.699273:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776400.699276:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776400.699280:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776400.699314:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776400.699321:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776400.699323:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776400.699327:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776400.699331:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776400.699338:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13283 00000400:00000200:3.0:1775776400.699347:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776400.699352:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776400.699355:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776400.699359:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776400.699362:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776400.699364:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776400.699367:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776400.699371:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776400.699373:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776400.699375:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776400.699378:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776400.699380:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776400.699382:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776400.699385:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776400.699387:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776400.699389:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776400.699391:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776400.699394:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776400.699397:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776400.699399:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776400.699403:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776400.699406:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776400.699409:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776400.699411:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776400.699426:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776400.699433:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775776400.699436:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776400.699439:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776400.699441:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776400.699444:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776400.699446:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776400.699450:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776400.699456:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000100:00000001:3.0:1775776400.699462:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776400.699470:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710411520/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776400.699480:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776400.699484:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776400.699488:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710411520/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776400.699507:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776400.699509:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776400.699512:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776400.699513:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776400.699515:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776400.699518:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776400.699520:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13283 00000400:00000200:3.0:1775776400.699524:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776400.699528:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776400.699531:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775776400.699533:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1775776400.699534:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776400.699537:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776400.699538:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776400.699541:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776400.699542:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776400.699545:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776400.699548:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776400.699550:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000040:0.0:1775776400.699551:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71500 x1862022710411520/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775776400.699552:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776400.699556:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776400.699560:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776400.699564:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775776400.699567:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776400.699570:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776400.699571:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776400.699575:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775776400.699576:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1775776400.699577:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:0.0:1775776400.699581:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776400.699581:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776400.699585:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:0.0:1775776400.699586:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776400.699589:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00100000:0.0:1775776400.699593:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710411520:192.168.201.115@tcp:250:kworker.0 00000100:00000001:3.0:1775776400.699594:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776400.699599:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776400.699599:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776400.699603:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776400.699604:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776400.699605:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776400.699609:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1775776400.699609:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776400.699613:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000010:0.0:1775776400.699615:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 00000008:00100000:3.0:1775776400.699617:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1775776400.699618:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775776400.699622:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776400.699624:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776400.699625:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776400.699630:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000100:00000010:3.0:1775776400.699630:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775776400.699634:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000400:00000010:0.0:1775776400.699635:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 02000000:00000001:3.0:1775776400.699636:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776400.699639:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775776400.699641:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710411520, portal 25 00000100:00000001:3.0:1775776400.699641:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776400.699643:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1775776400.699645:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1775776400.699646:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776400.699647:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849009408 : -123823860542208 : ffff8f6202c71500) 02000000:00000010:3.0:1775776400.699651:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000100:00000040:0.0:1775776400.699652:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71500 x1862022710411520/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776425 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776400.699657:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776400.699662:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776400.699666:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00100000:3.0:1775776400.699666:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710411648/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1775776400.699668:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776400.699672:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710411520, offset 0 00000100:00000001:3.0:1775776400.699680:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1775776400.699681:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776400.699684:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776400.699689:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775776400.699690:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710411648/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776400.699695:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776400.699701:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776400.699706:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:3.0:1775776400.699708:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776400.699711:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00080000:3.0:1775776400.699712:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775776400.699716:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00080000:3.0:1775776400.699718:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776400.699720:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776400.699723:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1775776400.699724:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453767:453767:256:4294967295] 192.168.201.115@tcp LPNI seq info [453692:453692:8:4294967295] 00000100:00000001:3.0:1775776400.699727:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776400.699731:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13283 00000400:00000200:0.0:1775776400.699733:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776400.699737:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776400.699741:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776400.699744:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776400.699748:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776400.699750:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776400.699752:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776400.699755:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776400.699760:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:0.0:1775776400.699761:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000020:00000040:3.0:1775776400.699762:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776400.699765:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776400.699768:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776400.699771:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:0.0:1775776400.699774:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000020:00000001:3.0:1775776400.699775:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776400.699781:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000800:00000010:0.0:1775776400.699782:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000100:00000001:3.0:1775776400.699783:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776400.699787:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776400.699789:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776400.699792:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000800:00000200:0.0:1775776400.699793:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776400.699797:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000100:00080000:3.0:1775776400.699798:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776400.699804:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775776400.699807:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:3.0:1775776400.699808:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:0.1:1775776400.699812:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 02000000:00000001:3.0:1775776400.699813:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:0.0:1775776400.699817:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 02000000:00000001:3.0:1775776400.699818:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775776400.699821:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000100:00000001:3.0:1775776400.699822:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:0.0:1775776400.699825:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00100000:3.0:1775776400.699825:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776400.699830:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776400.699832:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1775776400.699835:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776400.699837:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000100:00000010:3.0:1775776400.699839:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000400:00000010:0.0:1775776400.699840:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000020:00000040:3.0:1775776400.699844:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000100:00000001:0.0:1775776400.699845:0:2411:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1775776400.699849:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000200:0.0:1775776400.699850:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71500 x1862022710411520/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776425 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775776400.699851:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776400.699856:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776400.699858:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776400.699861:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776400.699864:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000010:3.0:1775776400.699866:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000100:00000040:0.0:1775776400.699867:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71500 x1862022710411520/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776425 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776400.699871:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776400.699876:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776400.699879:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:3.0:1775776400.699880:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710411776/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776400.699883:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776400.699891:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776400.699895:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307959558 00000100:00000001:3.0:1775776400.699896:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776400.699900:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776400.699901:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3480. 00000100:00000040:3.0:1775776400.699905:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710411776/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.699908:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776400.699915:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307959558 00000100:00000001:3.0:1775776400.699915:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776400.699919:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3d80. 00000100:00000040:3.0:1775776400.699920:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776400.699923:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000040:3.0:1775776400.699926:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1775776400.699927:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1775776400.699930:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776400.699931:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776400.699934:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000040:3.0:1775776400.699937:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00100000:0.0:1775776400.699940:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776400/real 1775776400] req@ffff8f6202c71500 x1862022710411520/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776400.699944:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13293) 00000400:00000001:0.0:1775776400.699950:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776400.699951:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776400.699953:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776400.699955:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776400.699956:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776400.699958:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776400.699961:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71500 x1862022710411520/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776400.699967:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71500 x1862022710411520/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.699971:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776400.699972:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776400.699973:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776400.699974:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776400.699976:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71500 x1862022710411520/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.699980:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776400.699982:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71500 x1862022710411520/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.699986:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776400.699988:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776400.699989:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776400.699989:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776400.699991:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776400.699998:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776400.700000:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776400.700002:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776400.700004:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776400.700006:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71500 x1862022710411520/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776400.700011:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710411520:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776400.700014:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776400.700015:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776400.700016:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71500 x1862022710411520/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.700024:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776400.700026:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776400.700027:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775776400.700029:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776400.700030:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776400.700031:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776400.700032:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776400.700034:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35c00. 02000000:00000001:0.0:1775776400.700036:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776400.700037:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776400.700038:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71500. 00000100:00000001:0.0:1775776400.700040:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776400.700041:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776400.700042:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776400.700047:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776400.700048:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776400.700050:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776400.700050:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776400.700051:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776400.700053:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70380 x1862022710411648/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.700057:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776400.700058:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776400.700059:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776400.700061:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776400.700062:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776400.700063:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710411648:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776400.700065:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776400.700067:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776400.700068:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776400.700069:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776400.700070:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775776400.700072:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776400.700074:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776400.700076:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776400.700078:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776400.700080:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710411648, portal 10 00000100:00000001:0.0:1775776400.700081:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776400.700081:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004928 : -123823860546688 : ffff8f6202c70380) 00000100:00000040:0.0:1775776400.700086:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70380 x1862022710411648/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776425 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.700089:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776400.700091:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776400.700092:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776400.700093:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710411648, offset 0 00000400:00000200:0.0:1775776400.700095:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776400.700097:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776400.700099:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776400.700101:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776400.700102:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776400.700104:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776400.700105:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776400.700107:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453768:453768:256:4294967295] 192.168.201.115@tcp LPNI seq info [453693:453693:8:4294967295] 00000400:00000200:0.0:1775776400.700110:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776400.700113:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776400.700116:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776400.700117:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776400.700119:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776400.700121:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775776400.700123:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776400.700125:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1775776400.700127:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776400.700128:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776400.700130:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776400.700150:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776400.700153:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776400.700155:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776400.700158:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776400.700172:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70380 x1862022710411648/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776425 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.700180:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776400.700183:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70380 x1862022710411648/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776425 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.700190:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776400.700191:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776400.700194:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776400.700198:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776400.700201:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776400.700203:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307955724 00000100:00000001:0.0:1775776400.700205:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307959558 00000100:00000001:0.0:1775776400.700206:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307955724 00000100:00000001:0.0:1775776400.700208:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307959558 00000100:00000001:0.0:1775776400.700209:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776400.700210:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776400.700211:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776400.700214:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776400/real 1775776400] req@ffff8f6202c70380 x1862022710411648/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776400.700218:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776400.700219:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776400.700220:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776400.700222:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776400.700223:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776400.700224:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776400.700226:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70380 x1862022710411648/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776400.700231:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70380 x1862022710411648/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.700242:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776400.700243:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776400.700245:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776400.700246:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776400.700249:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70380 x1862022710411648/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.700255:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776400.700259:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70380 x1862022710411648/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.700266:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776400.700268:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776400.700270:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776400.700271:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776400.700273:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776400.700277:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776400.700279:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776400.700282:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776400.700284:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776400.700288:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70380 x1862022710411648/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776400.700296:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710411648:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776400.700299:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776400.700302:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73480 x1862022710411776/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.700308:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776400.700309:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776400.700312:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776400.700313:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776400.700315:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776400.700317:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710411776:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776400.700320:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776400.700322:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776400.700324:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776400.700332:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776400.700334:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776400.700336:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776400.700338:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776400.700340:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776400.700342:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776400.700344:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710411776, portal 10 00000100:00000001:0.0:1775776400.700345:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776400.700346:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849017472 : -123823860534144 : ffff8f6202c73480) 00000100:00000040:0.0:1775776400.700350:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73480 x1862022710411776/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776425 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.700354:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776400.700355:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776400.700356:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776400.700357:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710411776, offset 0 00000400:00000200:0.0:1775776400.700359:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776400.700362:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776400.700364:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776400.700365:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776400.700367:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776400.700369:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776400.700370:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776400.700372:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453769:453769:256:4294967295] 192.168.201.115@tcp LPNI seq info [453694:453694:8:4294967295] 00000400:00000200:0.0:1775776400.700374:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776400.700378:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776400.700381:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776400.700383:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776400.700385:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776400.700387:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775776400.700390:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776400.700391:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1775776400.700393:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776400.700395:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776400.700397:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776400.700399:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776400.700400:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776400.700401:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776400.700403:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776400.700405:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73480 x1862022710411776/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776425 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.700409:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776400.700411:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73480 x1862022710411776/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776425 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.700420:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776400.700421:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776400.700423:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776400.700427:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776400.700429:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776400.700430:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307955724 00000100:00000001:0.0:1775776400.700431:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307959558 00000100:00000001:0.0:1775776400.700433:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307955724 00000100:00000001:0.0:1775776400.700434:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307959558 00000100:00000001:0.0:1775776400.700435:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776400.700437:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776400.700438:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776400.700440:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776400/real 1775776400] req@ffff8f6202c73480 x1862022710411776/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776400.700445:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776400.700446:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776400.700447:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776400.700448:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776400.700449:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776400.700450:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776400.700452:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73480 x1862022710411776/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776400.700456:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73480 x1862022710411776/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.700460:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776400.700461:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776400.700462:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776400.700462:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776400.700464:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73480 x1862022710411776/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.700467:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776400.700469:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73480 x1862022710411776/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.700473:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776400.700474:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776400.700475:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776400.700475:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776400.700476:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776400.700477:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776400.700479:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776400.700480:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776400.700481:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776400.700484:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73480 x1862022710411776/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776400.700488:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710411776:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776400.700490:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776400.700491:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776400.700493:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70380 x1862022710411648/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.700496:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776400.700497:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776400.700498:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775776400.700499:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776400.700500:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776400.700501:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776400.700502:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776400.700502:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34000. 02000000:00000001:0.0:1775776400.700504:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776400.700504:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776400.700505:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70380. 00000100:00000001:0.0:1775776400.700507:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776400.700507:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776400.700508:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776400.700510:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73480 x1862022710411776/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776425 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776400.700513:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776400.700514:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776400.700515:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776400.700515:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776400.700516:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776400.700517:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776400.700517:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776400.700518:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36c00. 02000000:00000001:0.0:1775776400.700519:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776400.700520:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776400.700521:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73480. 00000100:00000001:0.0:1775776400.700522:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776400.700523:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776400.700524:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776400.700526:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776400.700527:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776400.700528:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776400.700529:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776400.827254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776400.827259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776400.827263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776400.827266:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776400.827267:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776400.827269:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776400.827271:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776400.827273:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776400.827274:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776400.827276:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776400.827278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776400.827279:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776400.827280:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776400.827281:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776400.827282:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776400.827282:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776400.827283:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776400.827284:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776400.827334:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776400.827336:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776400.827338:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776401.723233:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776401.723240:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776401.723241:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776401.723247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776401.723250:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776401.723253:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776401.723256:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776401.723258:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776401.723260:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776401.851267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776401.851273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776401.851278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776401.851280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776401.851284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776401.851285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776401.851339:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776401.851343:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776401.851345:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776402.747246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776402.747253:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776402.747254:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776402.747258:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776402.747261:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776402.747264:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776402.747267:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776402.747268:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776402.747270:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776402.875235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776402.875239:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776402.875243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776402.875244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776402.875246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776402.875247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776402.875317:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776402.875320:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776402.875323:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776403.771221:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776403.771228:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776403.771230:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776403.771236:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776403.771239:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776403.771243:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776403.771245:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776403.771248:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776403.771251:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776403.899252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776403.899257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776403.899261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776403.899263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776403.899267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776403.899268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776403.899350:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776403.899355:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776403.899358:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776404.603224:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307959558, at 4307959616 00000100:00000010:1.0:1775776404.603237:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3d80. 00000100:00000040:0.0:1775776404.603249:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307959558, at 4307959616 00000100:00000010:0.0:1775776404.603258:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776404.795317:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776404.795329:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776404.795331:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776404.795339:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776404.795343:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776404.795349:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776404.795351:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776404.795354:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776404.795357:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776404.923226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776404.923231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776404.923237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776404.923238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776404.923240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776404.923241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776404.923287:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776404.923291:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776404.923295:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776405.819250:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775776405.819253:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776405.819258:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776405.819260:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776405.819263:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1775776405.819265:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776405.819267:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775776405.819269:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776405.819271:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776405.819273:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776405.819274:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775776405.819276:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776405.819279:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776405.819281:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776405.819284:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13288 00000400:00000200:3.0:1775776405.819296:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776405.819304:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776405.819308:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776405.819311:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776405.819315:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776405.819318:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776405.819320:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776405.819325:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776405.819327:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776405.819330:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776405.819333:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776405.819335:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776405.819338:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776405.819340:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776405.819342:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776405.819344:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776405.819346:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776405.819349:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776405.819352:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776405.819355:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776405.819359:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776405.819369:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776405.819371:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776405.819373:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776405.819375:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776405.819385:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775776405.819388:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776405.819391:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776405.819393:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776405.819395:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776405.819397:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776405.819402:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776405.819412:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000100:00000001:3.0:1775776405.819420:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776405.819430:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710411904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776405.819441:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776405.819447:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776405.819452:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710411904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776405.819475:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776405.819478:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776405.819481:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776405.819482:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776405.819484:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776405.819486:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776405.819488:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13288 00000400:00000200:3.0:1775776405.819492:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776405.819494:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776405.819496:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775776405.819497:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1775776405.819500:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776405.819501:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776405.819502:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776405.819503:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776405.819506:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776405.819508:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776405.819510:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776405.819512:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000040:0.0:1775776405.819513:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71c00 x1862022710411904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775776405.819513:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776405.819516:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776405.819519:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776405.819521:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776405.819527:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776405.819530:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776405.819533:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776405.819535:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776405.819536:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:0.0:1775776405.819538:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776405.819539:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776405.819543:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776405.819544:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776405.819545:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776405.819545:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776405.819550:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776405.819552:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776405.819554:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776405.819555:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776405.819559:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00100000:0.0:1775776405.819563:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710411904:192.168.201.115@tcp:250:kworker.0 00000008:00100000:3.0:1775776405.819563:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776405.819567:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776405.819570:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1775776405.819570:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776405.819572:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000020:00000040:3.0:1775776405.819575:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776405.819576:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776405.819576:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776405.819578:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776405.819580:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776405.819581:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776405.819581:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776405.819584:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776405.819586:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776405.819587:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000001:3.0:1775776405.819590:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776405.819593:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 00000100:00100000:3.0:1775776405.819594:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710412032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1775776405.819598:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776405.819605:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776405.819607:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776405.819608:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775776405.819612:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710412032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1775776405.819614:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776405.819621:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000001:3.0:1775776405.819623:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776405.819627:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:0.0:1775776405.819629:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710411904, portal 25 00000100:00000001:0.0:1775776405.819634:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1775776405.819634:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776405.819636:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849011200 : -123823860540416 : ffff8f6202c71c00) 00000100:00000001:3.0:1775776405.819637:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776405.819640:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:0.0:1775776405.819645:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71c00 x1862022710411904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776430 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776405.819645:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776405.819651:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13288 00000100:00000001:0.0:1775776405.819655:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1775776405.819659:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:0.0:1775776405.819660:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776405.819663:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776405.819667:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710411904, offset 0 00000400:00000200:3.0:1775776405.819667:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776405.819670:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776405.819675:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776405.819678:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776405.819679:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776405.819682:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:0.0:1775776405.819683:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000020:00000001:3.0:1775776405.819684:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776405.819686:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776405.819688:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776405.819690:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776405.819692:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:0.0:1775776405.819695:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000001:3.0:1775776405.819696:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776405.819700:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:0.0:1775776405.819703:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:3.0:1775776405.819704:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776405.819708:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:0.0:1775776405.819710:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1775776405.819711:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776405.819715:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:0.0:1775776405.819717:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00080000:3.0:1775776405.819721:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776405.819724:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776405.819726:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1775776405.819727:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 02000000:00000001:3.0:1775776405.819729:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775776405.819733:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 02000000:00000001:3.0:1775776405.819734:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776405.819739:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453770:453770:256:4294967295] 192.168.201.115@tcp LPNI seq info [453695:453695:8:4294967295] 00000100:00000001:3.0:1775776405.819745:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:0.0:1775776405.819746:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000008:00100000:3.0:1775776405.819749:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776405.819753:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776405.819755:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776405.819760:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000400:00000200:0.0:1775776405.819762:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000020:00000040:3.0:1775776405.819763:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776405.819768:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776405.819769:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776405.819771:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776405.819773:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776405.819775:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776405.819778:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000800:00000200:0.0:1775776405.819781:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776405.819782:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776405.819788:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710412160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:0.0:1775776405.819790:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000100:00000001:3.0:1775776405.819796:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776405.819797:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776405.819800:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710412160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1775776405.819805:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775776405.819808:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776405.819810:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776405.819813:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776405.819815:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.0:1775776405.819816:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000100:00000040:3.0:1775776405.819818:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:0.1:1775776405.819827:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776405.819831:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13298) 00000800:00000200:0.1:1775776405.819831:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1775776405.819838:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776405.819841:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776405.819849:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776405.819854:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776405.819860:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776405.819863:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776405.819868:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776405.819872:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71c00 x1862022710411904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776430 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.819882:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776405.819886:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71c00 x1862022710411904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776430 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.819894:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776405.819896:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776405.819901:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776405.819905:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776405.819908:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307964678 00000100:00000010:0.0:1775776405.819914:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776405.819922:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776405.819929:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307964678 00000100:00000010:0.0:1775776405.819932:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3780. 00000100:00000001:0.0:1775776405.819935:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776405.819938:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776405.819940:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776405.819943:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776405.819955:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776405/real 1775776405] req@ffff8f6202c71c00 x1862022710411904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776405.819964:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776405.819966:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776405.819968:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776405.819971:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776405.819972:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776405.819975:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776405.819978:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71c00 x1862022710411904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776405.819992:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71c00 x1862022710411904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.819998:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776405.819999:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776405.820001:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776405.820003:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776405.820006:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71c00 x1862022710411904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.820012:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776405.820017:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71c00 x1862022710411904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.820024:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776405.820027:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776405.820028:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776405.820029:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776405.820031:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776405.820033:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776405.820035:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776405.820039:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776405.820042:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776405.820045:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71c00 x1862022710411904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776405.820053:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710411904:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776405.820058:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776405.820060:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776405.820062:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71c00 x1862022710411904/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.820100:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776405.820103:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776405.820106:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775776405.820109:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776405.820111:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776405.820112:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776405.820114:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776405.820117:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35000. 02000000:00000001:0.0:1775776405.820121:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776405.820123:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776405.820124:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71c00. 00000100:00000001:0.0:1775776405.820128:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776405.820129:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776405.820131:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776405.820156:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776405.820158:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776405.820160:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776405.820161:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776405.820163:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776405.820166:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70e00 x1862022710412032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.820172:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776405.820174:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776405.820176:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776405.820177:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776405.820180:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776405.820182:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710412032:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776405.820185:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776405.820187:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776405.820188:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776405.820190:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776405.820193:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775776405.820194:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776405.820198:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776405.820200:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776405.820204:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776405.820207:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710412032, portal 10 00000100:00000001:0.0:1775776405.820208:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776405.820209:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849007616 : -123823860544000 : ffff8f6202c70e00) 00000100:00000040:0.0:1775776405.820216:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70e00 x1862022710412032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776430 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.820222:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776405.820224:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776405.820226:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776405.820228:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710412032, offset 0 00000400:00000200:0.0:1775776405.820231:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776405.820235:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776405.820238:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776405.820241:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776405.820243:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776405.820246:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776405.820248:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776405.820251:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453771:453771:256:4294967295] 192.168.201.115@tcp LPNI seq info [453696:453696:8:4294967295] 00000400:00000200:0.0:1775776405.820255:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776405.820261:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776405.820265:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776405.820267:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776405.820271:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776405.820274:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775776405.820278:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776405.820281:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1775776405.820284:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776405.820286:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776405.820289:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776405.820292:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776405.820295:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776405.820297:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776405.820299:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776405.820302:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70e00 x1862022710412032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776430 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.820309:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776405.820311:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70e00 x1862022710412032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776430 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.820317:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776405.820318:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776405.820321:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776405.820325:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776405.820327:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776405.820329:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307960844 00000100:00000001:0.0:1775776405.820332:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307964678 00000100:00000001:0.0:1775776405.820341:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307960844 00000100:00000001:0.0:1775776405.820346:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307964678 00000100:00000001:0.0:1775776405.820348:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776405.820350:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776405.820352:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776405.820355:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776405/real 1775776405] req@ffff8f6202c70e00 x1862022710412032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776405.820362:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776405.820364:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776405.820365:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776405.820368:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776405.820369:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776405.820371:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776405.820374:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70e00 x1862022710412032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776405.820381:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70e00 x1862022710412032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.820393:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776405.820394:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776405.820395:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776405.820397:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776405.820400:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70e00 x1862022710412032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.820412:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776405.820416:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70e00 x1862022710412032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.820422:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776405.820424:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776405.820425:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776405.820427:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776405.820429:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776405.820434:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776405.820436:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776405.820439:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776405.820441:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776405.820445:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70e00 x1862022710412032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776405.820452:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710412032:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776405.820455:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776405.820458:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70000 x1862022710412160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.820463:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776405.820465:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776405.820467:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776405.820468:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776405.820471:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776405.820473:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710412160:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776405.820477:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776405.820479:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776405.820480:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776405.820482:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776405.820486:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776405.820488:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776405.820492:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776405.820494:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776405.820497:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776405.820500:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710412160, portal 10 00000100:00000001:0.0:1775776405.820502:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776405.820503:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004032 : -123823860547584 : ffff8f6202c70000) 00000100:00000040:0.0:1775776405.820507:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70000 x1862022710412160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776430 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.820514:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776405.820516:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776405.820518:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776405.820520:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710412160, offset 0 00000400:00000200:0.0:1775776405.820523:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776405.820527:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776405.820530:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776405.820532:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776405.820535:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776405.820537:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776405.820539:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776405.820542:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453772:453772:256:4294967295] 192.168.201.115@tcp LPNI seq info [453697:453697:8:4294967295] 00000400:00000200:0.0:1775776405.820546:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776405.820551:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776405.820555:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776405.820558:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776405.820561:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776405.820564:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1775776405.820568:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776405.820570:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1775776405.820574:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776405.820576:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776405.820580:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776405.820583:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776405.820586:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776405.820587:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776405.820590:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776405.820593:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70000 x1862022710412160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776430 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.820599:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776405.820602:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70000 x1862022710412160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776430 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.820607:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776405.820609:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776405.820611:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776405.820613:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776405.820616:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776405.820618:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307960844 00000100:00000001:0.0:1775776405.820620:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307964678 00000100:00000001:0.0:1775776405.820622:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307960844 00000100:00000001:0.0:1775776405.820624:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307964678 00000100:00000001:0.0:1775776405.820626:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776405.820628:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776405.820630:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776405.820633:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776405/real 1775776405] req@ffff8f6202c70000 x1862022710412160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776405.820640:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776405.820642:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776405.820643:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776405.820645:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776405.820647:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776405.820648:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776405.820651:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70000 x1862022710412160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776405.820658:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70000 x1862022710412160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.820664:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776405.820665:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776405.820667:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776405.820668:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776405.820672:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70000 x1862022710412160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.820678:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776405.820682:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70000 x1862022710412160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.820689:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776405.820691:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776405.820693:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776405.820694:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776405.820695:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776405.820697:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776405.820699:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776405.820701:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776405.820702:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776405.820705:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70000 x1862022710412160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776405.820710:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710412160:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776405.820712:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776405.820713:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776405.820715:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70e00 x1862022710412032/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.820718:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776405.820719:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776405.820721:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775776405.820722:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776405.820723:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776405.820724:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776405.820725:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776405.820726:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37800. 02000000:00000001:0.0:1775776405.820728:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776405.820729:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776405.820730:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70e00. 00000100:00000001:0.0:1775776405.820732:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776405.820733:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776405.820734:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776405.820735:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70000 x1862022710412160/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776430 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776405.820738:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776405.820739:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776405.820740:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776405.820741:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776405.820741:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776405.820742:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776405.820743:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776405.820744:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37c00. 02000000:00000001:0.0:1775776405.820744:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776405.820746:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776405.820747:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70000. 00000100:00000001:0.0:1775776405.820748:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776405.820749:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776405.820750:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776405.820752:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776405.820753:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776405.820754:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776405.820756:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776405.947267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776405.947274:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776405.947276:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776405.947279:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776405.947284:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776405.947286:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776405.947287:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776405.947289:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776405.947291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776405.947292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776405.947295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776405.947295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776405.947357:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776405.947362:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776405.947366:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776406.843245:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776406.843251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776406.843252:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776406.843258:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776406.843262:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776406.843265:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776406.843268:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776406.843270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776406.843272:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776406.971251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776406.971257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776406.971262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776406.971263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776406.971267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776406.971269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776406.971332:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776406.971336:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776406.971339:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776407.867246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776407.867251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776407.867252:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776407.867256:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776407.867258:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776407.867261:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776407.867263:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776407.867265:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776407.867266:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776407.995217:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776407.995221:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776407.995225:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776407.995225:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776407.995228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776407.995229:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776407.995280:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776407.995284:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776407.995287:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776408.891228:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776408.891236:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776408.891239:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776408.891244:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776408.891249:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776408.891252:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776408.891256:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776408.891259:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776408.891262:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776409.019250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776409.019255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776409.019260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776409.019261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776409.019263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776409.019264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776409.019321:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776409.019324:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776409.019326:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776409.723220:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307964678, at 4307964736 00000100:00000010:1.0:1775776409.723232:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3780. 00000100:00000040:0.0:1775776409.723236:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307964678, at 4307964736 00000100:00000010:0.0:1775776409.723246:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776409.915281:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776409.915292:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776409.915294:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776409.915301:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776409.915305:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776409.915309:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776409.915311:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776409.915314:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776409.915317:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776410.043210:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776410.043217:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776410.043223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776410.043224:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776410.043228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776410.043230:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776410.043288:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776410.043292:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776410.043296:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776410.939252:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776410.939259:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776410.939261:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776410.939266:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776410.939269:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776410.939282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776410.939287:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776410.939289:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776410.939292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776410.939296:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776410.939304:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776410.939307:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776410.939311:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776410.939316:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776410.939325:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13293 00000400:00000200:2.0:1775776410.939335:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776410.939341:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776410.939344:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776410.939346:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776410.939351:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776410.939353:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776410.939355:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776410.939358:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776410.939360:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776410.939363:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776410.939365:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776410.939367:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776410.939369:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776410.939371:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776410.939372:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776410.939374:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776410.939375:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776410.939378:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776410.939380:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776410.939382:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776410.939385:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776410.939388:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776410.939390:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776410.939392:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776410.939394:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776410.939405:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775776410.939408:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776410.939412:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776410.939413:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776410.939415:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776410.939418:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776410.939422:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776410.939431:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775776410.939438:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776410.939447:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710412288/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776410.939467:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776410.939472:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776410.939476:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710412288/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776410.939501:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776410.939505:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776410.939509:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776410.939511:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776410.939512:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776410.939516:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776410.939520:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13293 00000400:00000200:2.0:1775776410.939527:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776410.939534:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776410.939537:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775776410.939541:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1775776410.939542:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776410.939546:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776410.939547:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776410.939549:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775776410.939553:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776410.939556:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:0.0:1775776410.939563:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344bb80 x1862022710412288/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775776410.939570:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776410.939574:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:0.0:1775776410.939577:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1775776410.939579:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775776410.939580:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775776410.939584:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 02000000:00000001:0.0:1775776410.939587:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1775776410.939591:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:0.0:1775776410.939594:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1775776410.939598:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:0.0:1775776410.939602:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776410.939608:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00100000:0.0:1775776410.939611:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710412288:192.168.201.115@tcp:250:kworker.0 00000100:00000001:2.0:1775776410.939613:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776410.939620:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776410.939621:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1775776410.939623:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776410.939627:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:0.0:1775776410.939630:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1775776410.939631:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776410.939637:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776410.939638:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776410.939640:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776410.939645:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1775776410.939647:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776410.939650:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776410.939652:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000010:0.0:1775776410.939655:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4400. 00000008:00100000:2.0:1775776410.939657:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1775776410.939660:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775776410.939664:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776410.939669:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776410.939677:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000010:2.0:1775776410.939681:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000400:00000010:0.0:1775776410.939685:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000020:00000040:2.0:1775776410.939687:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776410.939692:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776410.939693:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776410.939694:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000001:2.0:1775776410.939699:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776410.939702:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:0.0:1775776410.939704:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710412288, portal 25 02000000:00000001:2.0:1775776410.939705:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776410.939712:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776410.939715:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662550400 : -123823047001216 : ffff8f623344bb80) 00000100:00000040:0.0:1775776410.939724:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344bb80 x1862022710412288/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776435 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1775776410.939729:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:0.0:1775776410.939738:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1775776410.939741:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776410.939748:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776410.939753:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00100000:2.0:1775776410.939758:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710412416/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1775776410.939759:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710412288, offset 0 00000100:00000001:2.0:1775776410.939776:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776410.939783:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776410.939785:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000040:2.0:1775776410.939789:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710412416/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776410.939798:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776410.939799:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00080000:2.0:1775776410.939804:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776410.939808:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776410.939809:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776410.939811:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1775776410.939817:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:2.0:1775776410.939818:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776410.939823:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13293 00000400:00000200:0.0:1775776410.939827:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775776410.939831:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776410.939836:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776410.939843:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775776410.939846:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000400:00000200:0.0:1775776410.939852:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00080000:2.0:1775776410.939853:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776410.939861:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776410.939867:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775776410.939871:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453773:453773:256:4294967295] 192.168.201.115@tcp LPNI seq info [453698:453698:8:4294967295] 00000100:00000040:2.0:1775776410.939873:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776410.939881:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776410.939887:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776410.939892:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776410.939896:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:0.0:1775776410.939897:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000040:2.0:1775776410.939899:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776410.939902:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776410.939905:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776410.939908:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776410.939910:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776410.939917:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776410.939925:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776410.939931:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:0.0:1775776410.939932:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00080000:2.0:1775776410.939940:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776410.939948:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776410.939950:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776410.939954:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:0.0:1775776410.939955:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 02000000:00000001:2.0:1775776410.939964:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776410.939967:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776410.939969:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000010:0.0:1775776410.939970:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000008:00000001:2.0:1775776410.939971:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776410.939978:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776410.939982:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000800:00000200:0.0:1775776410.940002:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000040:2.0:1775776410.940006:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000800:00000200:0.0:1775776410.940008:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 02000000:00000001:2.0:1775776410.940009:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776410.940013:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776410.940021:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776410.940023:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776410.940028:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775776410.940031:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000010:2.0:1775776410.940035:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000800:00000200:0.1:1775776410.940037:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000100:00000001:2.0:1775776410.940039:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776410.940050:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710412544/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775776410.940054:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776410.940062:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000100:00000001:2.0:1775776410.940067:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776410.940070:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776410.940074:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:2.0:1775776410.940075:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710412544/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776410.940092:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1775776410.940097:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776410.940100:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775776410.940101:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000100:00000040:2.0:1775776410.940105:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000010:0.0:1775776410.940107:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000040:2.0:1775776410.940109:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776410.940114:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1775776410.940115:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:2.0:1775776410.940120:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13303) 00000100:00000200:0.0:1775776410.940125:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344bb80 x1862022710412288/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776435 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940165:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776410.940169:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344bb80 x1862022710412288/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776435 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940177:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776410.940179:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776410.940185:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776410.940190:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776410.940193:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307969799 00000100:00000010:0.0:1775776410.940199:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776410.940205:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776410.940213:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307969799 00000100:00000010:0.0:1775776410.940216:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3f60. 00000100:00000001:0.0:1775776410.940219:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776410.940223:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776410.940225:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776410.940228:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776410.940233:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776410/real 1775776410] req@ffff8f623344bb80 x1862022710412288/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776410.940243:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776410.940244:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776410.940247:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776410.940250:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776410.940251:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776410.940254:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776410.940258:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344bb80 x1862022710412288/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776410.940273:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344bb80 x1862022710412288/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940281:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776410.940283:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776410.940285:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776410.940288:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776410.940293:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344bb80 x1862022710412288/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940302:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776410.940307:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344bb80 x1862022710412288/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940316:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776410.940319:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776410.940320:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776410.940322:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776410.940324:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776410.940326:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776410.940330:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776410.940334:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776410.940336:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776410.940340:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344bb80 x1862022710412288/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776410.940346:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710412288:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776410.940349:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776410.940357:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776410.940359:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344bb80 x1862022710412288/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940368:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776410.940370:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776410.940372:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776410.940375:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776410.940377:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776410.940379:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776410.940382:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776410.940384:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4d400. 02000000:00000001:0.0:1775776410.940387:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776410.940389:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776410.940391:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344bb80. 00000100:00000001:0.0:1775776410.940395:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776410.940396:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776410.940399:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776410.940405:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776410.940406:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776410.940408:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776410.940409:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776410.940410:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776410.940413:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233448700 x1862022710412416/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940418:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776410.940419:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776410.940420:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776410.940422:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776410.940423:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776410.940425:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710412416:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776410.940427:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776410.940429:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776410.940430:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776410.940431:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776410.940434:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776410.940435:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776410.940438:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776410.940440:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776410.940443:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776410.940445:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710412416, portal 10 00000100:00000001:0.0:1775776410.940446:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776410.940447:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662536960 : -123823047014656 : ffff8f6233448700) 00000100:00000040:0.0:1775776410.940452:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233448700 x1862022710412416/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776435 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940456:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776410.940458:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776410.940459:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776410.940461:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710412416, offset 0 00000400:00000200:0.0:1775776410.940464:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776410.940466:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776410.940469:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776410.940471:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776410.940473:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776410.940475:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776410.940477:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776410.940479:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453774:453774:256:4294967295] 192.168.201.115@tcp LPNI seq info [453699:453699:8:4294967295] 00000400:00000200:0.0:1775776410.940481:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776410.940486:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776410.940489:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776410.940491:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776410.940493:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776410.940495:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775776410.940498:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776410.940500:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1775776410.940503:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776410.940504:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776410.940506:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776410.940509:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776410.940511:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776410.940512:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776410.940515:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776410.940517:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233448700 x1862022710412416/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776435 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940522:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776410.940524:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448700 x1862022710412416/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776435 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940528:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776410.940529:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776410.940531:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776410.940534:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776410.940536:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776410.940538:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307965964 00000100:00000001:0.0:1775776410.940540:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307969799 00000100:00000001:0.0:1775776410.940541:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307965964 00000100:00000001:0.0:1775776410.940543:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307969799 00000100:00000001:0.0:1775776410.940544:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776410.940545:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776410.940547:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776410.940549:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776410/real 1775776410] req@ffff8f6233448700 x1862022710412416/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776410.940553:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776410.940554:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776410.940555:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776410.940557:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776410.940558:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776410.940559:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776410.940561:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233448700 x1862022710412416/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776410.940566:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233448700 x1862022710412416/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940574:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776410.940575:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776410.940576:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776410.940577:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776410.940579:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233448700 x1862022710412416/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940583:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776410.940585:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233448700 x1862022710412416/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940589:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776410.940591:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776410.940591:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776410.940592:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776410.940593:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776410.940597:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776410.940599:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776410.940601:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776410.940602:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776410.940604:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233448700 x1862022710412416/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776410.940609:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710412416:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776410.940611:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776410.940613:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449c00 x1862022710412544/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940618:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776410.940619:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776410.940621:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776410.940622:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776410.940623:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776410.940625:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710412544:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776410.940627:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776410.940629:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776410.940630:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776410.940631:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776410.940633:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775776410.940634:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776410.940636:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776410.940637:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776410.940639:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776410.940641:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710412544, portal 10 00000100:00000001:0.0:1775776410.940642:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776410.940643:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662542336 : -123823047009280 : ffff8f6233449c00) 00000100:00000040:0.0:1775776410.940647:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449c00 x1862022710412544/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776435 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940651:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776410.940653:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776410.940654:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776410.940655:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710412544, offset 0 00000400:00000200:0.0:1775776410.940658:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776410.940660:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776410.940662:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776410.940663:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776410.940665:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776410.940666:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776410.940668:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776410.940670:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453775:453775:256:4294967295] 192.168.201.115@tcp LPNI seq info [453700:453700:8:4294967295] 00000400:00000200:0.0:1775776410.940673:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776410.940677:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776410.940680:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776410.940682:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776410.940684:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776410.940686:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1775776410.940688:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776410.940690:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1775776410.940692:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776410.940693:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776410.940695:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776410.940697:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776410.940699:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776410.940700:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776410.940701:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776410.940703:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449c00 x1862022710412544/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776435 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940708:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776410.940709:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449c00 x1862022710412544/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776435 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940713:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776410.940714:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776410.940716:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776410.940718:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776410.940720:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776410.940721:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307965964 00000100:00000001:0.0:1775776410.940723:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307969799 00000100:00000001:0.0:1775776410.940724:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307965964 00000100:00000001:0.0:1775776410.940726:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307969799 00000100:00000001:0.0:1775776410.940727:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776410.940728:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776410.940729:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776410.940731:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776410/real 1775776410] req@ffff8f6233449c00 x1862022710412544/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776410.940737:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776410.940738:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776410.940739:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776410.940741:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776410.940741:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776410.940743:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776410.940744:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449c00 x1862022710412544/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776410.940749:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449c00 x1862022710412544/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940753:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776410.940753:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776410.940754:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776410.940755:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776410.940757:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449c00 x1862022710412544/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940761:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776410.940763:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449c00 x1862022710412544/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940767:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776410.940768:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776410.940769:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776410.940770:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776410.940771:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776410.940772:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776410.940774:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776410.940776:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776410.940778:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776410.940781:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449c00 x1862022710412544/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776410.940786:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710412544:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776410.940789:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776410.940790:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776410.940791:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233448700 x1862022710412416/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940795:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776410.940796:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776410.940797:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776410.940798:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776410.940799:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776410.940800:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776410.940801:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776410.940802:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4cc00. 02000000:00000001:0.0:1775776410.940804:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776410.940804:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776410.940805:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233448700. 00000100:00000001:0.0:1775776410.940807:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776410.940808:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776410.940808:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776410.940810:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449c00 x1862022710412544/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776435 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776410.940814:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776410.940815:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776410.940816:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775776410.940818:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776410.940819:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776410.940819:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776410.940820:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776410.940821:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4d800. 02000000:00000001:0.0:1775776410.940822:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776410.940824:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776410.940825:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449c00. 00000100:00000001:0.0:1775776410.940826:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776410.940827:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776410.940828:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776410.940830:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776410.940831:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776410.940832:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776410.940833:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776411.067221:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776411.067226:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776411.067230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776411.067233:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776411.067235:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776411.067238:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776411.067240:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776411.067242:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776411.067243:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776411.067245:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776411.067247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776411.067248:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776411.067249:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776411.067250:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776411.067251:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776411.067252:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776411.067253:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776411.067254:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776411.067324:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776411.067329:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776411.067332:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1775776411.771290:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:0.0:1775776411.771298:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:0.0:1775776411.771303:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1775776411.771476:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776411.771493:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1775776411.771495:0:2405:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1775776411.771497:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776411.771502:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776411.963234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776411.963241:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776411.963244:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776411.963251:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776411.963255:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776411.963258:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776411.963262:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776411.963265:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776411.963276:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776412.091215:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776412.091219:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776412.091223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776412.091224:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776412.091227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776412.091228:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776412.091272:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776412.091277:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776412.091280:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776412.987235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776412.987240:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776412.987242:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776412.987246:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776412.987249:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776412.987252:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776412.987254:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776412.987256:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776412.987258:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776413.115254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776413.115259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776413.115263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776413.115264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776413.115273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776413.115274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776413.115346:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776413.115350:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776413.115354:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776414.011239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776414.011244:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776414.011245:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776414.011250:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776414.011253:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776414.011256:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776414.011259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776414.011260:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776414.011262:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776414.139292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776414.139300:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776414.139307:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776414.139309:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776414.139313:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776414.139314:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776414.139381:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776414.139385:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776414.139389:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776414.843281:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307969799, at 4307969856 00000100:00000040:0.0:1775776414.843301:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307969799, at 4307969856 00000100:00000010:1.0:1775776414.843308:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3f60. 00000100:00000010:0.0:1775776414.843312:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776415.035274:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776415.035282:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776415.035284:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776415.035290:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776415.035293:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776415.035298:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776415.035301:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776415.035303:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776415.035306:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776415.163253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776415.163257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776415.163262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776415.163263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776415.163266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776415.163267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776415.163345:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776415.163349:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776415.163352:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775776416.059250:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776416.059258:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776416.059260:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775776416.059264:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1775776416.059264:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776416.059270:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776416.059272:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775776416.059274:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776416.059279:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:2.0:1775776416.059281:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13298 00000100:00000001:0.0:1775776416.059284:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776416.059289:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776416.059292:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:2.0:1775776416.059295:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776416.059296:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776416.059301:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776416.059305:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776416.059309:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776416.059313:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776416.059316:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776416.059320:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776416.059322:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776416.059327:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776416.059329:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776416.059331:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776416.059335:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776416.059338:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776416.059341:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776416.059345:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776416.059347:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776416.059350:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776416.059353:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776416.059356:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776416.059360:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776416.059363:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776416.059366:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776416.059369:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776416.059372:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776416.059374:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776416.059377:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776416.059388:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775776416.059404:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776416.059409:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776416.059411:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776416.059414:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776416.059416:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776416.059420:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776416.059430:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775776416.059437:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776416.059447:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710412672/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776416.059455:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776416.059460:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776416.059465:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710412672/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776416.059494:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776416.059498:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776416.059502:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776416.059503:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776416.059505:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776416.059508:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776416.059511:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13298 00000400:00000200:2.0:1775776416.059516:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776416.059519:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776416.059522:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776416.059525:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776416.059527:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776416.059530:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776416.059532:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776416.059534:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776416.059536:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776416.059538:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776416.059540:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776416.059541:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776416.059544:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776416.059547:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776416.059550:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776416.059552:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776416.059554:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776416.059556:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776416.059559:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776416.059562:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776416.059563:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776416.059569:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776416.059571:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776416.059573:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776416.059578:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776416.059581:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776416.059583:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776416.059587:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775776416.059603:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776416.059606:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776416.059606:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776416.059608:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776416.059609:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776416.059610:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776416.059613:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775776416.059616:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776416.059620:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710412800/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776416.059626:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776416.059627:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776416.059630:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710412800/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776416.059639:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776416.059642:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776416.059645:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776416.059647:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776416.059649:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776416.059652:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776416.059655:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13298 00000400:00000200:2.0:1775776416.059659:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776416.059662:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776416.059665:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776416.059668:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776416.059671:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776416.059673:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776416.059675:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776416.059678:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776416.059679:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776416.059681:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776416.059683:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776416.059685:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776416.059687:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776416.059691:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776416.059693:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776416.059695:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776416.059697:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776416.059713:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776416.059717:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776416.059721:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776416.059723:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776416.059727:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776416.059729:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776416.059731:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776416.059733:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776416.059736:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776416.059738:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776416.059742:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775776416.059745:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776416.059747:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776416.059749:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776416.059750:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776416.059752:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776416.059754:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776416.059758:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775776416.059761:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776416.059766:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710412928/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776416.059775:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776416.059778:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776416.059782:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710412928/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776416.059791:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776416.059794:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776416.059798:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776416.059801:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776416.059804:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776416.059809:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13308) 00000100:00000001:0.0:1775776416.060038:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776416.060042:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776416.060044:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776416.060050:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449180 x1862022710412672/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060059:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776416.060061:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776416.060064:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776416.060068:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776416.060071:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776416.060075:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710412672:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776416.060079:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776416.060083:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776416.060086:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776416.060088:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776416.060094:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775776416.060097:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776416.060102:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776416.060107:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776416.060113:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776416.060121:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710412672, portal 25 00000100:00000001:0.0:1775776416.060124:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776416.060125:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662539648 : -123823047011968 : ffff8f6233449180) 00000100:00000040:0.0:1775776416.060132:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449180 x1862022710412672/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060154:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776416.060157:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776416.060159:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776416.060162:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710412672, offset 0 00000400:00000200:0.0:1775776416.060167:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776416.060173:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776416.060177:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776416.060181:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776416.060184:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776416.060188:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776416.060190:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776416.060193:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453776:453776:256:4294967295] 192.168.201.115@tcp LPNI seq info [453701:453701:8:4294967295] 00000400:00000200:0.0:1775776416.060197:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776416.060204:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776416.060209:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776416.060214:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776416.060219:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776416.060222:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775776416.060226:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776416.060229:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1775776416.060232:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776416.060234:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776416.060239:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776416.060242:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776416.060246:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776416.060247:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776416.060250:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776416.060253:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449180 x1862022710412672/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060260:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776416.060263:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449180 x1862022710412672/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776441 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060271:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776416.060272:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776416.060278:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776416.060282:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776416.060285:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307974919 00000100:00000010:0.0:1775776416.060290:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776416.060297:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776416.060304:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307974919 00000100:00000010:0.0:1775776416.060307:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3420. 00000100:00000001:0.0:1775776416.060310:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776416.060313:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776416.060316:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776416.060318:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776416.060323:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776416/real 1775776416] req@ffff8f6233449180 x1862022710412672/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776416.060340:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776416.060342:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776416.060344:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776416.060346:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776416.060348:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776416.060351:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776416.060355:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449180 x1862022710412672/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776416.060364:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449180 x1862022710412672/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060370:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776416.060372:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776416.060374:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776416.060375:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776416.060378:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449180 x1862022710412672/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060386:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776416.060399:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449180 x1862022710412672/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060408:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776416.060411:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776416.060413:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776416.060414:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776416.060416:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776416.060418:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776416.060421:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776416.060426:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776416.060428:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776416.060433:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449180 x1862022710412672/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776416.060442:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710412672:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776416.060447:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776416.060450:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344ad80 x1862022710412800/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060457:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776416.060459:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776416.060461:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776416.060463:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776416.060465:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776416.060467:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710412800:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776416.060470:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776416.060471:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776416.060473:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776416.060474:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776416.060476:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776416.060478:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776416.060481:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776416.060483:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776416.060486:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776416.060488:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710412800, portal 10 00000100:00000001:0.0:1775776416.060489:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776416.060490:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662546816 : -123823047004800 : ffff8f623344ad80) 00000100:00000040:0.0:1775776416.060496:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344ad80 x1862022710412800/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060500:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776416.060502:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776416.060503:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776416.060504:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710412800, offset 0 00000400:00000200:0.0:1775776416.060507:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776416.060511:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776416.060513:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776416.060515:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776416.060517:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776416.060519:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776416.060521:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776416.060523:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453777:453777:256:4294967295] 192.168.201.115@tcp LPNI seq info [453702:453702:8:4294967295] 00000400:00000200:0.0:1775776416.060526:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776416.060530:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776416.060533:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776416.060535:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776416.060538:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776416.060540:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775776416.060543:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776416.060545:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1775776416.060548:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776416.060549:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776416.060551:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776416.060553:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776416.060555:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776416.060556:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776416.060558:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776416.060560:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344ad80 x1862022710412800/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060566:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776416.060568:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344ad80 x1862022710412800/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776441 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060572:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776416.060573:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776416.060575:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776416.060578:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776416.060580:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776416.060581:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307971084 00000100:00000001:0.0:1775776416.060583:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307974919 00000100:00000001:0.0:1775776416.060585:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307971084 00000100:00000001:0.0:1775776416.060586:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307974919 00000100:00000001:0.0:1775776416.060587:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776416.060589:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776416.060590:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776416.060592:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776416/real 1775776416] req@ffff8f623344ad80 x1862022710412800/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776416.060597:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776416.060598:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776416.060599:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776416.060600:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776416.060601:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776416.060603:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776416.060605:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344ad80 x1862022710412800/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776416.060609:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344ad80 x1862022710412800/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060614:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776416.060615:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776416.060616:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776416.060617:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776416.060619:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344ad80 x1862022710412800/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060623:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776416.060625:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344ad80 x1862022710412800/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060629:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776416.060632:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776416.060633:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776416.060634:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776416.060635:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776416.060638:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776416.060640:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776416.060642:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776416.060643:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776416.060645:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344ad80 x1862022710412800/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776416.060650:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710412800:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776416.060653:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776416.060654:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449500 x1862022710412928/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060658:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776416.060659:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776416.060660:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776416.060661:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776416.060663:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776416.060664:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710412928:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776416.060666:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776416.060667:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776416.060668:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776416.060669:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776416.060671:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775776416.060672:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776416.060674:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776416.060676:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776416.060678:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776416.060679:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710412928, portal 10 00000100:00000001:0.0:1775776416.060681:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776416.060682:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662540544 : -123823047011072 : ffff8f6233449500) 00000100:00000040:0.0:1775776416.060686:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449500 x1862022710412928/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060690:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776416.060691:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776416.060693:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776416.060694:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710412928, offset 0 00000400:00000200:0.0:1775776416.060697:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776416.060699:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776416.060701:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776416.060702:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776416.060704:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776416.060706:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776416.060707:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776416.060709:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453778:453778:256:4294967295] 192.168.201.115@tcp LPNI seq info [453703:453703:8:4294967295] 00000400:00000200:0.0:1775776416.060712:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776416.060716:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776416.060719:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776416.060721:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776416.060723:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776416.060725:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1775776416.060727:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776416.060729:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1775776416.060732:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776416.060733:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776416.060734:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776416.060736:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776416.060738:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776416.060739:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776416.060741:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776416.060743:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449500 x1862022710412928/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060752:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776416.060754:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449500 x1862022710412928/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776441 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060758:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776416.060759:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776416.060760:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776416.060762:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776416.060764:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776416.060765:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307971084 00000100:00000001:0.0:1775776416.060766:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307974919 00000100:00000001:0.0:1775776416.060767:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307971084 00000100:00000001:0.0:1775776416.060769:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307974919 00000100:00000001:0.0:1775776416.060770:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776416.060771:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776416.060773:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776416.060775:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776416/real 1775776416] req@ffff8f6233449500 x1862022710412928/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776416.060779:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776416.060780:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776416.060781:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776416.060782:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776416.060783:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776416.060784:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776416.060786:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449500 x1862022710412928/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776416.060791:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449500 x1862022710412928/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060795:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776416.060797:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776416.060798:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776416.060799:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776416.060801:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449500 x1862022710412928/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060805:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776416.060807:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449500 x1862022710412928/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060811:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776416.060812:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776416.060813:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776416.060814:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776416.060815:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776416.060816:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776416.060818:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776416.060820:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776416.060821:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776416.060823:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449500 x1862022710412928/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776416.060828:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710412928:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776416.060830:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776416.060831:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776416.060833:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449180 x1862022710412672/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060837:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776416.060839:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776416.060841:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:0.0:1775776416.060842:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776416.060844:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776416.060845:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776416.060846:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776416.060848:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4e000. 02000000:00000001:0.0:1775776416.060851:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776416.060852:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776416.060854:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449180. 00000100:00000001:0.0:1775776416.060856:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776416.060857:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776416.060858:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776416.060860:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344ad80 x1862022710412800/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060864:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776416.060864:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776416.060865:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776416.060866:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776416.060867:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776416.060868:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776416.060869:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776416.060869:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4e800. 02000000:00000001:0.0:1775776416.060871:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776416.060871:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776416.060872:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344ad80. 00000100:00000001:0.0:1775776416.060875:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776416.060875:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776416.060876:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776416.060878:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449500 x1862022710412928/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776441 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776416.060882:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776416.060882:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776416.060883:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775776416.060884:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776416.060885:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776416.060886:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776416.060887:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776416.060887:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4c000. 02000000:00000001:0.0:1775776416.060889:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776416.060891:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776416.060892:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449500. 00000100:00000001:0.0:1775776416.060893:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776416.060894:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776416.060913:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776416.060920:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776416.060922:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776416.060924:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776416.060926:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776416.187241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776416.187249:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776416.187252:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776416.187256:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776416.187261:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776416.187263:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776416.187266:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776416.187268:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776416.187272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776416.187274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776416.187278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776416.187279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776416.187376:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776416.187379:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776416.187382:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776417.083340:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776417.083348:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776417.083351:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776417.083357:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776417.083362:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776417.083366:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776417.083370:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776417.083372:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776417.083375:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776417.211288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776417.211296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776417.211303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776417.211304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776417.211308:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776417.211309:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776417.211395:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776417.211400:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776417.211405:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776418.107272:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776418.107280:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776418.107283:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776418.107289:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776418.107292:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776418.107296:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776418.107298:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776418.107301:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776418.107303:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776418.235292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776418.235298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776418.235303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776418.235304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776418.235307:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776418.235308:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776418.235386:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776418.235392:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776418.235396:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776419.131286:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776419.131292:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776419.131294:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776419.131299:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776419.131302:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776419.131306:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776419.131308:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776419.131310:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776419.131312:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776419.259271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776419.259278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776419.259282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776419.259283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776419.259286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776419.259287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776419.259352:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776419.259355:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776419.259358:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776419.963251:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307974919, at 4307974976 00000100:00000010:1.0:1775776419.963265:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3420. 00000100:00000040:0.0:1775776419.963307:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307974919, at 4307974976 00000100:00000010:0.0:1775776419.963322:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776420.155288:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776420.155297:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776420.155299:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776420.155306:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776420.155310:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776420.155315:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776420.155318:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776420.155320:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776420.155323:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776420.283228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776420.283233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776420.283238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776420.283239:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776420.283243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776420.283245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776420.283307:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776420.283312:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776420.283315:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776421.179218:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776421.179222:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775776421.179226:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776421.179227:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776421.179229:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776421.179229:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776421.179234:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775776421.179235:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776421.179240:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776421.179240:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776421.179244:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776421.179248:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775776421.179248:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13303 00000100:00000001:2.0:1775776421.179252:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1775776421.179258:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:2.0:1775776421.179264:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776421.179266:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776421.179270:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776421.179274:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776421.179277:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776421.179279:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776421.179282:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776421.179285:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776421.179287:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776421.179290:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776421.179293:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776421.179295:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776421.179297:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776421.179300:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776421.179301:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776421.179303:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776421.179304:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776421.179307:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776421.179310:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776421.179312:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776421.179315:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776421.179318:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776421.179321:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776421.179324:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776421.179326:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776421.179335:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73100. 00000020:00000040:3.0:1775776421.179338:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776421.179342:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776421.179343:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776421.179345:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776421.179347:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776421.179350:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776421.179357:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:3.0:1775776421.179361:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776421.179368:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73100 x1862022710413056/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776421.179375:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776421.179378:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776421.179381:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f67e6552] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73100 x1862022710413056/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776421.179395:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776421.179397:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776421.179400:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776421.179400:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776421.179402:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776421.179403:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776421.179405:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13303 00000400:00000200:3.0:1775776421.179408:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776421.179410:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776421.179415:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776421.179418:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776421.179420:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776421.179421:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1775776421.179421:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776421.179424:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:0.0:1775776421.179426:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1775776421.179427:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776421.179429:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:0.0:1775776421.179430:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1775776421.179431:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776421.179434:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776421.179435:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776421.179437:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775776421.179439:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73100 x1862022710413056/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776421.179441:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776421.179444:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776421.179447:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776421.179450:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776421.179454:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000001:0.0:1775776421.179459:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775776421.179460:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776421.179462:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776421.179463:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776421.179467:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776421.179469:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776421.179471:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776421.179473:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776421.179474:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776421.179476:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775776421.179479:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775776421.179480:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776421.179484:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776421.179485:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710413056:192.168.201.115@tcp:250:kworker.0 00000100:00000001:3.0:1775776421.179486:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776421.179492:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000010:3.0:1775776421.179492:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775776421.179496:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776421.179497:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776421.179498:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776421.179500:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776421.179502:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776421.179502:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776421.179504:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776421.179507:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776421.179508:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776421.179510:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000001:3.0:1775776421.179512:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776421.179514:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6400. 00000100:00100000:3.0:1775776421.179517:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710413184/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1775776421.179519:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776421.179526:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1775776421.179531:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776421.179531:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776421.179537:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000100:00000040:3.0:1775776421.179537:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710413184/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776421.179545:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776421.179547:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00080000:3.0:1775776421.179547:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776421.179550:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776421.179552:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776421.179553:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776421.179555:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000200:0.0:1775776421.179557:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710413056, portal 25 00000100:00080000:3.0:1775776421.179557:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13303 00000400:00000200:3.0:1775776421.179561:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776421.179563:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000400:00000200:3.0:1775776421.179564:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:0.0:1775776421.179567:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849016576 : -123823860535040 : ffff8f6202c73100) 00000100:00080000:3.0:1775776421.179567:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776421.179570:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776421.179573:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776421.179574:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776421.179576:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776421.179578:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776421.179579:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000100:00000040:0.0:1775776421.179580:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73100 x1862022710413056/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775776421.179581:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776421.179583:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776421.179586:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776421.179589:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776421.179595:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776421.179598:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776421.179600:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1775776421.179600:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776421.179602:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776421.179605:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00000040:0.0:1775776421.179606:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00080000:3.0:1775776421.179609:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000400:00000010:0.0:1775776421.179611:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000001:3.0:1775776421.179612:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776421.179616:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:0.0:1775776421.179618:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710413056, offset 0 02000000:00000001:3.0:1775776421.179618:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776421.179622:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776421.179624:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776421.179625:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776421.179627:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776421.179628:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776421.179632:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000010:3.0:1775776421.179634:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000020:00000040:3.0:1775776421.179636:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776421.179637:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776421.179638:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776421.179639:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776421.179639:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776421.179640:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776421.179642:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775776421.179644:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776421.179647:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710413312/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775776421.179655:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776421.179655:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776421.179657:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776421.179661:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710413312/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776421.179666:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:3.0:1775776421.179670:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776421.179672:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775776421.179674:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1775776421.179676:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776421.179679:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775776421.179680:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000040:3.0:1775776421.179682:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776421.179686:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13313) 00000400:00000200:0.0:1775776421.179687:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776421.179691:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776421.179696:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453779:453779:256:4294967295] 192.168.201.115@tcp LPNI seq info [453704:453704:8:4294967295] 00000400:00000200:0.0:1775776421.179701:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776421.179710:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776421.179715:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776421.179721:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776421.179727:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776421.179731:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1775776421.179737:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776421.179741:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 23 secs later) 00000800:00000100:0.0:1775776421.179746:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776421.179748:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776421.179755:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776421.179759:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776421.179763:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776421.179766:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776421.179771:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776421.179775:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73100 x1862022710413056/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.179783:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776421.179786:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73100 x1862022710413056/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776446 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.179792:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776421.179793:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776421.179798:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776421.179802:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776421.179804:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307980038 00000100:00000010:0.0:1775776421.179833:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776421.179839:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776421.179844:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307980038 00000100:00000010:0.0:1775776421.179847:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3420. 00000100:00000001:0.0:1775776421.179850:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776421.179853:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776421.179856:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776421.179858:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776421.179861:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776421/real 1775776421] req@ffff8f6202c73100 x1862022710413056/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776421.179870:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776421.179871:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776421.179873:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776421.179875:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776421.179876:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776421.179879:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776421.179881:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73100 x1862022710413056/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776421.179888:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73100 x1862022710413056/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.179894:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776421.179895:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776421.179897:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776421.179898:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776421.179901:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73100 x1862022710413056/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.179908:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776421.179911:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73100 x1862022710413056/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.179917:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776421.179919:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776421.179921:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776421.179922:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776421.179924:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776421.179925:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776421.179928:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776421.179932:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776421.179934:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776421.179937:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73100 x1862022710413056/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776421.179944:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710413056:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776421.179948:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776421.179949:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776421.179952:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73100 x1862022710413056/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.179958:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776421.179960:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776421.179962:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775776421.179965:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776421.179967:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776421.179977:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776421.179979:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776421.179982:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35400. 02000000:00000001:0.0:1775776421.179985:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776421.179987:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776421.179988:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73100. 00000100:00000001:0.0:1775776421.179991:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776421.179992:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776421.179994:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776421.180000:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776421.180001:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776421.180007:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776421.180008:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776421.180009:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776421.180012:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70000 x1862022710413184/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.180018:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776421.180019:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776421.180021:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776421.180022:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776421.180024:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776421.180026:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710413184:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776421.180029:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776421.180031:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776421.180032:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776421.180034:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776421.180037:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775776421.180038:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776421.180042:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776421.180044:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776421.180047:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776421.180050:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710413184, portal 10 00000100:00000001:0.0:1775776421.180052:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776421.180053:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004032 : -123823860547584 : ffff8f6202c70000) 00000100:00000040:0.0:1775776421.180059:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70000 x1862022710413184/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.180064:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776421.180067:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776421.180069:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776421.180071:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710413184, offset 0 00000400:00000200:0.0:1775776421.180074:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776421.180078:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776421.180081:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776421.180084:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776421.180086:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776421.180089:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776421.180091:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776421.180094:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453780:453780:256:4294967295] 192.168.201.115@tcp LPNI seq info [453705:453705:8:4294967295] 00000400:00000200:0.0:1775776421.180098:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776421.180103:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776421.180107:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776421.180110:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776421.180113:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776421.180115:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1775776421.180119:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776421.180121:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 23 secs later) 00000800:00000100:0.0:1775776421.180125:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776421.180127:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776421.180129:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776421.180151:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776421.180154:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776421.180155:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776421.180158:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776421.180161:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70000 x1862022710413184/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.180168:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776421.180171:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70000 x1862022710413184/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776446 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.180176:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776421.180177:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776421.180180:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776421.180184:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776421.180187:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776421.180189:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307976204 00000100:00000001:0.0:1775776421.180191:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307980038 00000100:00000001:0.0:1775776421.180193:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307976204 00000100:00000001:0.0:1775776421.180195:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307980038 00000100:00000001:0.0:1775776421.180196:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776421.180198:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776421.180200:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776421.180203:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776421/real 1775776421] req@ffff8f6202c70000 x1862022710413184/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776421.180209:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776421.180211:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776421.180213:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776421.180215:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776421.180216:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776421.180218:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776421.180221:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70000 x1862022710413184/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776421.180227:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70000 x1862022710413184/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.180232:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776421.180233:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776421.180235:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776421.180236:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776421.180239:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70000 x1862022710413184/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.180244:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776421.180248:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70000 x1862022710413184/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.180253:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776421.180255:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776421.180257:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776421.180258:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776421.180260:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776421.180264:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776421.180266:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776421.180268:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776421.180270:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776421.180273:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70000 x1862022710413184/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776421.180280:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710413184:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776421.180283:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776421.180286:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70e00 x1862022710413312/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.180291:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776421.180293:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776421.180300:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776421.180302:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776421.180303:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776421.180306:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710413312:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776421.180308:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776421.180310:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776421.180311:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776421.180313:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776421.180316:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776421.180317:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776421.180320:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776421.180322:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776421.180325:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776421.180328:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710413312, portal 10 00000100:00000001:0.0:1775776421.180330:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776421.180331:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849007616 : -123823860544000 : ffff8f6202c70e00) 00000100:00000040:0.0:1775776421.180336:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70e00 x1862022710413312/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.180341:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776421.180343:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776421.180345:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776421.180347:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710413312, offset 0 00000400:00000200:0.0:1775776421.180350:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776421.180353:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776421.180356:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776421.180359:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776421.180361:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776421.180363:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776421.180365:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776421.180368:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453781:453781:256:4294967295] 192.168.201.115@tcp LPNI seq info [453706:453706:8:4294967295] 00000400:00000200:0.0:1775776421.180372:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776421.180377:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776421.180381:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776421.180384:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776421.180387:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776421.180389:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1775776421.180392:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776421.180394:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 23 secs later) 00000800:00000100:0.0:1775776421.180398:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776421.180399:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776421.180401:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776421.180405:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776421.180408:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776421.180410:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776421.180412:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776421.180415:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70e00 x1862022710413312/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.180421:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776421.180424:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70e00 x1862022710413312/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776446 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.180429:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776421.180430:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776421.180437:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776421.180440:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776421.180443:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776421.180445:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307976204 00000100:00000001:0.0:1775776421.180447:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307980038 00000100:00000001:0.0:1775776421.180449:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307976204 00000100:00000001:0.0:1775776421.180450:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307980038 00000100:00000001:0.0:1775776421.180452:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776421.180454:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776421.180456:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776421.180459:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776421/real 1775776421] req@ffff8f6202c70e00 x1862022710413312/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776421.180465:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776421.180466:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776421.180468:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776421.180470:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776421.180471:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776421.180473:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776421.180476:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70e00 x1862022710413312/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776421.180482:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70e00 x1862022710413312/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.180489:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776421.180490:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776421.180491:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776421.180492:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776421.180495:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70e00 x1862022710413312/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.180501:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776421.180504:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70e00 x1862022710413312/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.180510:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776421.180512:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776421.180513:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776421.180514:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776421.180516:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776421.180517:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776421.180520:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776421.180523:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776421.180525:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776421.180528:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70e00 x1862022710413312/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776421.180535:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710413312:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776421.180538:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776421.180539:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776421.180542:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70000 x1862022710413184/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.180548:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776421.180550:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776421.180551:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775776421.180554:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776421.180556:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776421.180557:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776421.180559:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776421.180560:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35800. 02000000:00000001:0.0:1775776421.180562:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776421.180563:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776421.180565:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70000. 00000100:00000001:0.0:1775776421.180567:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776421.180568:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776421.180570:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776421.180572:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70e00 x1862022710413312/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776446 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776421.180577:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776421.180578:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776421.180580:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776421.180581:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776421.180583:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776421.180584:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776421.180585:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776421.180587:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37400. 02000000:00000001:0.0:1775776421.180589:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776421.180590:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776421.180591:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70e00. 00000100:00000001:0.0:1775776421.180593:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776421.180594:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776421.180596:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776421.180599:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776421.180600:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776421.180602:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776421.180604:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776421.307288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776421.307295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776421.307299:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776421.307312:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776421.307314:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776421.307317:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776421.307319:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776421.307321:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776421.307322:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776421.307324:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776421.307328:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776421.307330:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776421.307331:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776421.307332:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776421.307333:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776421.307334:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776421.307336:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776421.307337:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776421.307434:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776421.307438:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776421.307440:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776422.203250:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776422.203258:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776422.203260:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776422.203265:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776422.203268:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776422.203272:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776422.203274:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776422.203276:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776422.203279:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776422.331272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776422.331281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776422.331286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776422.331287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776422.331289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776422.331290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776422.331351:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776422.331355:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776422.331359:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776423.227260:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776423.227267:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776423.227273:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776423.227279:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776423.227283:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776423.227287:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776423.227289:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776423.227291:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776423.227293:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776423.355308:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776423.355316:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776423.355323:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776423.355324:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776423.355328:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776423.355329:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776423.355409:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776423.355414:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776423.355418:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776424.251267:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776424.251273:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776424.251275:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776424.251280:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776424.251284:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776424.251288:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776424.251290:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776424.251292:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776424.251295:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776424.379253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776424.379263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776424.379271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776424.379272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776424.379277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776424.379278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776424.379334:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776424.379338:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776424.379341:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776425.083253:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307980038, at 4307980096 00000100:00000040:0.0:1775776425.083275:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307980038, at 4307980096 00000100:00000010:1.0:1775776425.083281:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3420. 00000100:00000010:0.0:1775776425.083285:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3480. 00000100:00000001:2.0:1775776425.275256:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776425.275262:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776425.275264:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776425.275269:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776425.275272:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776425.275276:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776425.275279:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776425.275281:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776425.275283:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776425.403265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776425.403274:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776425.403277:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776425.403282:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776425.403287:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776425.403289:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776425.403291:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776425.403293:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776425.403298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776425.403299:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776425.403303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776425.403304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776425.403363:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776425.403366:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776425.403369:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776426.299255:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775776426.299255:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776426.299262:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776426.299265:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776426.299267:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:2.0:1775776426.299270:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776426.299272:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775776426.299275:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776426.299277:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776426.299279:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776426.299281:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776426.299282:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776426.299285:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776426.299287:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776426.299289:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13308 00000400:00000200:3.0:1775776426.299299:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776426.299305:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776426.299308:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776426.299311:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776426.299313:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776426.299315:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776426.299317:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776426.299320:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776426.299321:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776426.299324:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776426.299326:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776426.299327:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776426.299329:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776426.299331:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776426.299332:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776426.299340:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776426.299342:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776426.299347:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776426.299349:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776426.299351:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776426.299354:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776426.299356:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776426.299358:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776426.299360:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776426.299362:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776426.299371:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775776426.299374:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776426.299377:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776426.299378:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776426.299381:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776426.299383:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776426.299387:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776426.299394:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000001:3.0:1775776426.299400:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776426.299409:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710413440/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776426.299420:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776426.299425:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776426.299431:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710413440/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776426.299451:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776426.299455:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776426.299458:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776426.299459:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776426.299461:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776426.299463:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776426.299466:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13308 00000400:00000200:3.0:1775776426.299471:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776426.299474:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776426.299477:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775776426.299479:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1775776426.299480:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776426.299483:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776426.299484:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776426.299487:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776426.299494:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000040:0.0:1775776426.299496:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71c00 x1862022710413440/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776426.299497:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776426.299502:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776426.299505:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776426.299508:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776426.299512:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:0.0:1775776426.299513:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776426.299515:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775776426.299516:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:0.0:1775776426.299520:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1775776426.299520:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:0.0:1775776426.299524:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1775776426.299526:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 02000000:00000001:0.0:1775776426.299529:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776426.299530:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776426.299534:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00100000:0.0:1775776426.299536:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710413440:192.168.201.115@tcp:250:kworker.0 00000100:00000001:3.0:1775776426.299537:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776426.299542:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000001:0.0:1775776426.299544:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1775776426.299546:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776426.299548:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776426.299550:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776426.299550:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776426.299554:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776426.299557:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776426.299558:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1775776426.299559:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776426.299561:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000010:0.0:1775776426.299564:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4400. 00000008:00100000:3.0:1775776426.299565:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1775776426.299569:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775776426.299569:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776426.299574:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776426.299577:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000400:00000200:0.0:1775776426.299581:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000040:3.0:1775776426.299581:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776426.299585:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000010:0.0:1775776426.299587:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 02000000:00000001:3.0:1775776426.299587:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776426.299594:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776426.299595:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:0.0:1775776426.299596:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 02000000:00000001:3.0:1775776426.299596:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776426.299603:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000100:00000001:3.0:1775776426.299605:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775776426.299606:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710413440, portal 25 00000100:00000001:0.0:1775776426.299609:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00100000:3.0:1775776426.299610:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710413568/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776426.299612:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849011200 : -123823860540416 : ffff8f6202c71c00) 00000100:00000001:3.0:1775776426.299620:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:0.0:1775776426.299623:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71c00 x1862022710413440/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776426.299623:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776426.299628:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710413568/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776426.299649:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776426.299650:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1775776426.299652:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776426.299655:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776426.299658:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:0.0:1775776426.299659:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00080000:3.0:1775776426.299659:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776426.299663:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000400:00000010:0.0:1775776426.299665:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00080000:3.0:1775776426.299665:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13308 00000100:00000200:0.0:1775776426.299671:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710413440, offset 0 00000400:00000200:3.0:1775776426.299672:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776426.299676:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776426.299677:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776426.299680:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776426.299682:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775776426.299685:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776426.299685:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776426.299689:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776426.299690:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776426.299691:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776426.299692:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776426.299693:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776426.299694:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:0.0:1775776426.299698:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000001:3.0:1775776426.299699:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776426.299703:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776426.299704:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776426.299705:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776426.299707:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:0.0:1775776426.299709:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000020:00000040:3.0:1775776426.299710:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776426.299725:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000400:00000200:0.0:1775776426.299728:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1775776426.299729:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776426.299732:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1775776426.299735:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776426.299740:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 02000000:00000001:3.0:1775776426.299741:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775776426.299744:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 02000000:00000001:3.0:1775776426.299746:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776426.299749:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:0.0:1775776426.299752:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453782:453782:256:4294967295] 192.168.201.115@tcp LPNI seq info [453707:453707:8:4294967295] 00000008:00100000:3.0:1775776426.299753:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776426.299759:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776426.299763:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776426.299763:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776426.299770:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775776426.299773:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776426.299777:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776426.299779:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776426.299781:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1775776426.299782:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 02000000:00000001:3.0:1775776426.299783:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776426.299786:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776426.299790:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000800:00000200:0.0:1775776426.299794:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776426.299795:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776426.299801:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710413696/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:0.0:1775776426.299802:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000100:00000001:3.0:1775776426.299809:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776426.299810:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775776426.299814:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776426.299815:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710413696/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1775776426.299820:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00000001:3.0:1775776426.299822:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776426.299826:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.1:1775776426.299831:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776426.299832:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776426.299836:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.1:1775776426.299837:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00000040:3.0:1775776426.299840:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000100:0.0:1775776426.299845:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776426.299845:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13319) 00000800:00000010:0.0:1775776426.299850:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776426.299858:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776426.299863:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776426.299869:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776426.299872:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776426.299877:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776426.299881:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71c00 x1862022710413440/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.299891:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776426.299894:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71c00 x1862022710413440/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776451 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.299901:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776426.299904:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776426.299909:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776426.299913:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776426.299917:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307985158 00000100:00000010:0.0:1775776426.299922:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776426.299929:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776426.299936:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307985158 00000100:00000010:0.0:1775776426.299939:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3f60. 00000100:00000001:0.0:1775776426.299943:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776426.299946:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776426.299949:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776426.299951:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776426.299955:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776426/real 1775776426] req@ffff8f6202c71c00 x1862022710413440/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776426.299965:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776426.299967:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776426.299969:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776426.299972:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776426.299974:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776426.299976:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776426.299979:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71c00 x1862022710413440/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776426.299988:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71c00 x1862022710413440/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.299996:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776426.299997:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776426.300000:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776426.300002:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776426.300005:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71c00 x1862022710413440/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.300013:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776426.300018:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71c00 x1862022710413440/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.300025:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776426.300027:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776426.300029:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776426.300031:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776426.300034:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776426.300036:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776426.300039:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776426.300042:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776426.300045:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776426.300049:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71c00 x1862022710413440/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776426.300057:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710413440:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776426.300062:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776426.300064:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776426.300067:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71c00 x1862022710413440/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.300073:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776426.300076:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776426.300079:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776426.300081:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776426.300083:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776426.300084:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776426.300087:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776426.300089:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37000. 02000000:00000001:0.0:1775776426.300093:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776426.300095:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776426.300097:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71c00. 00000100:00000001:0.0:1775776426.300100:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776426.300101:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776426.300104:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776426.300111:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776426.300113:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776426.300115:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776426.300116:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776426.300117:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776426.300121:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73480 x1862022710413568/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.300129:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776426.300130:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776426.300156:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776426.300158:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776426.300160:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776426.300163:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710413568:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776426.300166:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776426.300168:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776426.300169:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776426.300171:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776426.300174:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776426.300176:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776426.300179:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776426.300181:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776426.300185:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776426.300187:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710413568, portal 10 00000100:00000001:0.0:1775776426.300189:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776426.300191:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849017472 : -123823860534144 : ffff8f6202c73480) 00000100:00000040:0.0:1775776426.300197:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73480 x1862022710413568/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.300203:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776426.300206:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776426.300208:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776426.300210:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710413568, offset 0 00000400:00000200:0.0:1775776426.300214:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776426.300217:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776426.300221:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776426.300223:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776426.300226:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776426.300228:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776426.300230:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776426.300234:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453783:453783:256:4294967295] 192.168.201.115@tcp LPNI seq info [453708:453708:8:4294967295] 00000400:00000200:0.0:1775776426.300238:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776426.300245:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776426.300249:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776426.300252:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776426.300255:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776426.300258:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775776426.300262:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776426.300265:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775776426.300268:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776426.300270:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776426.300273:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776426.300276:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776426.300279:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776426.300280:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776426.300282:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776426.300285:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73480 x1862022710413568/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.300293:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776426.300296:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73480 x1862022710413568/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776451 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.300302:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776426.300304:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776426.300306:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776426.300310:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776426.300313:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776426.300315:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307981324 00000100:00000001:0.0:1775776426.300318:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307985158 00000100:00000001:0.0:1775776426.300320:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307981324 00000100:00000001:0.0:1775776426.300322:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307985158 00000100:00000001:0.0:1775776426.300324:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776426.300327:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776426.300328:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776426.300331:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776426/real 1775776426] req@ffff8f6202c73480 x1862022710413568/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776426.300347:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776426.300348:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776426.300349:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776426.300351:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776426.300353:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776426.300355:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776426.300358:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73480 x1862022710413568/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776426.300365:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73480 x1862022710413568/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.300372:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776426.300373:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776426.300375:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776426.300376:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776426.300379:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73480 x1862022710413568/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.300386:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776426.300389:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73480 x1862022710413568/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.300395:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776426.300398:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776426.300399:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776426.300401:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776426.300402:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776426.300407:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776426.300409:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776426.300413:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776426.300414:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776426.300421:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73480 x1862022710413568/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776426.300429:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710413568:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776426.300432:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776426.300435:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70380 x1862022710413696/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.300442:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776426.300443:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776426.300446:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776426.300447:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776426.300449:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776426.300451:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710413696:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776426.300454:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776426.300456:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776426.300457:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776426.300459:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776426.300462:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775776426.300464:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776426.300466:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776426.300468:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776426.300471:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776426.300473:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710413696, portal 10 00000100:00000001:0.0:1775776426.300475:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776426.300477:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004928 : -123823860546688 : ffff8f6202c70380) 00000100:00000040:0.0:1775776426.300481:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70380 x1862022710413696/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.300487:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776426.300490:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776426.300492:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776426.300494:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710413696, offset 0 00000400:00000200:0.0:1775776426.300497:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776426.300501:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776426.300504:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776426.300507:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776426.300509:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776426.300511:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776426.300514:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776426.300517:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453784:453784:256:4294967295] 192.168.201.115@tcp LPNI seq info [453709:453709:8:4294967295] 00000400:00000200:0.0:1775776426.300521:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776426.300527:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776426.300531:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776426.300534:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776426.300537:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776426.300540:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775776426.300543:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776426.300546:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775776426.300550:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776426.300551:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776426.300554:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776426.300557:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776426.300559:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776426.300561:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776426.300563:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776426.300566:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70380 x1862022710413696/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.300574:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776426.300576:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70380 x1862022710413696/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776451 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.300583:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776426.300585:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776426.300587:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776426.300590:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776426.300593:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776426.300595:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307981324 00000100:00000001:0.0:1775776426.300597:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307985158 00000100:00000001:0.0:1775776426.300599:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307981324 00000100:00000001:0.0:1775776426.300601:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307985158 00000100:00000001:0.0:1775776426.300603:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776426.300605:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776426.300607:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776426.300610:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776426/real 1775776426] req@ffff8f6202c70380 x1862022710413696/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776426.300616:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776426.300618:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776426.300619:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776426.300622:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776426.300623:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776426.300625:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776426.300628:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70380 x1862022710413696/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776426.300635:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70380 x1862022710413696/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.300641:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776426.300643:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776426.300644:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776426.300645:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776426.300648:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70380 x1862022710413696/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.300654:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776426.300658:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70380 x1862022710413696/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.300665:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776426.300667:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776426.300669:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776426.300670:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776426.300671:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776426.300673:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776426.300675:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776426.300678:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776426.300680:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776426.300683:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70380 x1862022710413696/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776426.300690:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710413696:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776426.300693:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776426.300695:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776426.300698:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73480 x1862022710413568/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.300704:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776426.300705:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776426.300707:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776426.300709:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776426.300710:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776426.300711:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776426.300713:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776426.300714:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34400. 02000000:00000001:0.0:1775776426.300729:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776426.300730:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776426.300732:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73480. 00000100:00000001:0.0:1775776426.300735:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776426.300736:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776426.300738:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776426.300741:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70380 x1862022710413696/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776451 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776426.300746:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776426.300747:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776426.300749:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775776426.300751:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776426.300752:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776426.300753:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776426.300754:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776426.300756:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34800. 02000000:00000001:0.0:1775776426.300758:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776426.300759:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776426.300760:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70380. 00000100:00000001:0.0:1775776426.300762:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776426.300763:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776426.300765:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776426.300767:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776426.300769:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776426.300771:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776426.300772:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776426.427213:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776426.427219:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776426.427224:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776426.427225:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776426.427227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776426.427228:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776426.427278:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776426.427281:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776426.427284:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776427.325873:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776427.325882:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776427.325884:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776427.325892:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776427.325897:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776427.325902:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776427.325906:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776427.325909:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776427.325913:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776427.451255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776427.451261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776427.451266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776427.451268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776427.451271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776427.451272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776427.451344:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776427.451350:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776427.451355:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776428.347227:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776428.347237:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776428.347239:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776428.347245:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776428.347248:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776428.347252:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776428.347254:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776428.347257:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776428.347260:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776428.475273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776428.475281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776428.475287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776428.475289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776428.475292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776428.475294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776428.475353:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776428.475358:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776428.475361:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776429.371269:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776429.371276:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776429.371278:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776429.371286:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776429.371290:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776429.371294:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776429.371298:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776429.371301:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776429.371304:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776429.499225:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776429.499230:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776429.499233:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776429.499234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776429.499236:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776429.499237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776429.499300:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776429.499305:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776429.499309:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776430.203238:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307985158, at 4307985216 00000100:00000040:0.0:1775776430.203246:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307985158, at 4307985216 00000100:00000010:0.0:1775776430.203253:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3480. 00000100:00000010:1.0:1775776430.203253:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3f60. 00000100:00000001:0.0:1775776430.395228:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776430.395236:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776430.395238:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776430.395244:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776430.395247:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776430.395250:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776430.395253:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776430.395256:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776430.395258:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776430.523266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776430.523272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776430.523278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776430.523279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776430.523282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776430.523284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776430.523350:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776430.523354:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776430.523357:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776431.419269:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776431.419279:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1775776431.419279:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776431.419281:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776431.419288:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:0.0:1775776431.419289:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776431.419291:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775776431.419296:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1775776431.419298:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776431.419300:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776431.419302:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775776431.419304:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776431.419308:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775776431.419311:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13314 00000100:00000001:0.0:1775776431.419312:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776431.419321:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776431.419327:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776431.419330:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776431.419333:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776431.419336:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776431.419338:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776431.419340:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:1.0:1775776431.419343:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776431.419345:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:1.0:1775776431.419347:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776431.419350:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:1.0:1775776431.419351:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:1.0:1775776431.419353:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:1.0:1775776431.419355:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776431.419362:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776431.419363:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776431.419364:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:1.0:1775776431.419367:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:1.0:1775776431.419369:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776431.419371:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776431.419374:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776431.419376:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776431.419378:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776431.419379:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776431.419381:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776431.419389:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623491aa00. 00000020:00000040:1.0:1775776431.419392:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:1.0:1775776431.419396:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776431.419397:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776431.419400:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776431.419403:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776431.419408:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776431.419417:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4c800. 00000100:00000001:1.0:1775776431.419426:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776431.419442:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623491aa00 x1862022710413824/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776431.419453:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776431.419457:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776431.419460:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fa7ee57c] to pc [ptlrpcd_rcv+-1] req@ffff8f623491aa00 x1862022710413824/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776431.419475:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776431.419477:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776431.419480:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776431.419481:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776431.419482:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776431.419485:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776431.419487:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13314 00000400:00000200:1.0:1775776431.419490:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776431.419493:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776431.419495:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776431.419497:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776431.419498:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776431.419500:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776431.419502:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:1.0:1775776431.419503:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776431.419504:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:1.0:1775776431.419505:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775776431.419506:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1775776431.419507:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:1.0:1775776431.419510:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775776431.419512:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1775776431.419512:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775776431.419516:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776431.419516:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776431.419517:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776431.419520:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776431.419521:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:1.0:1775776431.419523:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000040:0.0:1775776431.419526:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491aa00 x1862022710413824/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775776431.419527:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776431.419531:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776431.419534:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776431.419539:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776431.419542:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1775776431.419542:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776431.419545:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776431.419546:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1775776431.419549:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1775776431.419551:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:1.0:1775776431.419552:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776431.419554:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776431.419555:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:1.0:1775776431.419557:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6234918a80. 02000000:00000001:0.0:1775776431.419560:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1775776431.419560:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776431.419565:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00100000:0.0:1775776431.419568:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710413824:192.168.201.115@tcp:250:kworker.0 02000000:00000001:1.0:1775776431.419568:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776431.419569:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776431.419571:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776431.419573:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776431.419576:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000010:1.0:1775776431.419577:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4f400. 00000100:00000001:1.0:1775776431.419581:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776431.419582:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:1.0:1775776431.419585:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6234918a80 x1862022710413952/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1775776431.419587:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776431.419592:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1775776431.419595:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776431.419597:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776431.419599:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000026c0c0d3] to pc [ptlrpcd_rcv+-1] req@ffff8f6234918a80 x1862022710413952/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1775776431.419600:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775776431.419605:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776431.419607:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776431.419610:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776431.419612:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:0.0:1775776431.419614:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776431.419614:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776431.419616:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776431.419619:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000400:00000010:0.0:1775776431.419621:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000100:00080000:1.0:1775776431.419621:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13314 00000400:00000200:1.0:1775776431.419625:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776431.419646:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000400:00000200:1.0:1775776431.419650:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776431.419652:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000200:0.0:1775776431.419655:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710413824, portal 25 00000100:00000040:1.0:1775776431.419655:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776431.419659:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776431.419660:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000040:1.0:1775776431.419661:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776431.419664:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684369408 : -123823025182208 : ffff8f623491aa00) 00000100:00000001:1.0:1775776431.419664:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:1.0:1775776431.419668:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776431.419669:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:1.0:1775776431.419671:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776431.419672:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:1.0:1775776431.419674:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775776431.419676:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491aa00 x1862022710413824/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775776431.419676:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:1.0:1775776431.419681:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776431.419684:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776431.419686:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776431.419690:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:1.0:1775776431.419693:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:1.0:1775776431.419697:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776431.419699:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1775776431.419700:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776431.419702:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776431.419704:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1775776431.419706:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 02000000:00000001:1.0:1775776431.419707:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776431.419710:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776431.419711:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000010:0.0:1775776431.419712:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000008:00000001:1.0:1775776431.419713:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776431.419716:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775776431.419717:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710413824, offset 0 00000100:00000010:1.0:1775776431.419719:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623491b800. 00000020:00000040:1.0:1775776431.419722:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776431.419723:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776431.419725:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776431.419727:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1775776431.419728:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 02000000:00000001:1.0:1775776431.419729:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776431.419732:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776431.419735:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4c000. 00000100:00000001:1.0:1775776431.419738:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776431.419742:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00100000:1.0:1775776431.419742:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623491b800 x1862022710414080/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776431.419749:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776431.419752:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776431.419753:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000040:1.0:1775776431.419757:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000958ac43a] to pc [ptlrpcd_rcv+-1] req@ffff8f623491b800 x1862022710414080/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776431.419762:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1775776431.419765:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776431.419768:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775776431.419771:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000040:1.0:1775776431.419772:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:1.0:1775776431.419775:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775776431.419779:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000040:1.0:1775776431.419779:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:1.0:1775776431.419785:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13324) 00000400:00000200:0.0:1775776431.419786:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776431.419793:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453785:453785:256:4294967295] 192.168.201.115@tcp LPNI seq info [453710:453710:8:4294967295] 00000400:00000200:0.0:1775776431.419798:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776431.419809:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776431.419816:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776431.419822:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776431.419829:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776431.419833:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775776431.419838:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776431.419842:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775776431.419846:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776431.419848:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776431.419855:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776431.419859:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776431.419864:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776431.419867:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776431.419872:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776431.419877:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491aa00 x1862022710413824/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.419887:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776431.419890:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491aa00 x1862022710413824/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776456 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.419897:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776431.419899:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776431.419906:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776431.419910:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776431.419913:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307990278 00000100:00000010:0.0:1775776431.419918:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776431.419926:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776431.419933:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307990278 00000100:00000010:0.0:1775776431.419936:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3780. 00000100:00000001:0.0:1775776431.419940:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776431.419942:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776431.419945:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776431.419948:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776431.419952:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776431/real 1775776431] req@ffff8f623491aa00 x1862022710413824/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776431.419962:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776431.419964:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776431.419966:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776431.419968:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776431.419970:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776431.419972:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776431.419976:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491aa00 x1862022710413824/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776431.419985:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491aa00 x1862022710413824/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.419991:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776431.419994:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776431.419996:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776431.419998:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776431.420002:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491aa00 x1862022710413824/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.420008:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776431.420012:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491aa00 x1862022710413824/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.420018:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776431.420020:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776431.420021:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776431.420022:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776431.420024:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776431.420025:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776431.420027:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776431.420030:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776431.420031:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776431.420034:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491aa00 x1862022710413824/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776431.420039:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710413824:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776431.420042:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776431.420043:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776431.420045:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491aa00 x1862022710413824/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.420048:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776431.420050:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776431.420052:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775776431.420055:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776431.420056:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776431.420057:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776431.420059:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776431.420060:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4c800. 02000000:00000001:0.0:1775776431.420064:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776431.420065:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776431.420067:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491aa00. 00000100:00000001:0.0:1775776431.420069:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776431.420070:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776431.420071:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776431.420078:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776431.420079:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776431.420080:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776431.420081:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776431.420082:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776431.420084:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918a80 x1862022710413952/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.420089:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776431.420090:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776431.420091:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776431.420092:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776431.420093:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776431.420095:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710413952:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776431.420097:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776431.420098:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776431.420099:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776431.420100:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776431.420102:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775776431.420103:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776431.420106:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776431.420108:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776431.420110:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776431.420112:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710413952, portal 10 00000100:00000001:0.0:1775776431.420113:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776431.420114:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684361344 : -123823025190272 : ffff8f6234918a80) 00000100:00000040:0.0:1775776431.420119:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918a80 x1862022710413952/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.420124:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776431.420125:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776431.420126:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776431.420128:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710413952, offset 0 00000400:00000200:0.0:1775776431.420130:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776431.420153:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776431.420157:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776431.420159:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776431.420162:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776431.420164:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776431.420167:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776431.420170:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453786:453786:256:4294967295] 192.168.201.115@tcp LPNI seq info [453711:453711:8:4294967295] 00000400:00000200:0.0:1775776431.420174:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776431.420180:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776431.420185:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776431.420189:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776431.420193:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776431.420195:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775776431.420197:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776431.420199:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775776431.420201:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776431.420203:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776431.420205:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776431.420207:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776431.420209:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776431.420210:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776431.420212:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776431.420214:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918a80 x1862022710413952/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.420219:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776431.420221:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918a80 x1862022710413952/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776456 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.420229:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776431.420230:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776431.420232:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776431.420235:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776431.420238:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776431.420239:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307986444 00000100:00000001:0.0:1775776431.420241:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307990278 00000100:00000001:0.0:1775776431.420242:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307986444 00000100:00000001:0.0:1775776431.420244:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307990278 00000100:00000001:0.0:1775776431.420245:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776431.420246:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776431.420247:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776431.420249:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776431/real 1775776431] req@ffff8f6234918a80 x1862022710413952/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776431.420254:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776431.420256:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776431.420257:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776431.420259:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776431.420261:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776431.420263:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776431.420265:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918a80 x1862022710413952/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776431.420273:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918a80 x1862022710413952/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.420279:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776431.420280:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776431.420282:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776431.420283:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776431.420286:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918a80 x1862022710413952/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.420293:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776431.420297:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918a80 x1862022710413952/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.420303:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776431.420305:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776431.420307:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776431.420308:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776431.420310:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776431.420315:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776431.420318:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776431.420321:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776431.420323:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776431.420327:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918a80 x1862022710413952/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776431.420334:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710413952:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776431.420338:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776431.420341:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491b800 x1862022710414080/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.420346:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776431.420347:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776431.420358:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776431.420360:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776431.420362:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776431.420364:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710414080:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776431.420367:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776431.420369:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776431.420371:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776431.420373:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776431.420376:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776431.420377:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776431.420380:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776431.420383:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776431.420386:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776431.420389:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710414080, portal 10 00000100:00000001:0.0:1775776431.420390:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776431.420392:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684372992 : -123823025178624 : ffff8f623491b800) 00000100:00000040:0.0:1775776431.420397:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491b800 x1862022710414080/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.420404:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776431.420406:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776431.420408:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776431.420410:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710414080, offset 0 00000400:00000200:0.0:1775776431.420414:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776431.420417:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776431.420420:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776431.420422:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776431.420423:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776431.420425:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776431.420426:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776431.420428:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453787:453787:256:4294967295] 192.168.201.115@tcp LPNI seq info [453712:453712:8:4294967295] 00000400:00000200:0.0:1775776431.420431:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776431.420434:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776431.420437:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776431.420439:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776431.420441:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776431.420443:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775776431.420445:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776431.420447:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775776431.420449:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776431.420451:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776431.420453:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776431.420455:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776431.420457:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776431.420458:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776431.420459:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776431.420461:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491b800 x1862022710414080/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.420467:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776431.420468:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491b800 x1862022710414080/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776456 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.420472:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776431.420473:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776431.420475:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776431.420477:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776431.420479:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776431.420480:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307986444 00000100:00000001:0.0:1775776431.420481:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307990278 00000100:00000001:0.0:1775776431.420483:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307986444 00000100:00000001:0.0:1775776431.420484:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307990278 00000100:00000001:0.0:1775776431.420486:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776431.420487:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776431.420488:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776431.420490:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776431/real 1775776431] req@ffff8f623491b800 x1862022710414080/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776431.420494:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776431.420495:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776431.420495:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776431.420497:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776431.420498:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776431.420499:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776431.420501:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491b800 x1862022710414080/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776431.420506:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491b800 x1862022710414080/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.420509:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776431.420510:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776431.420511:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776431.420511:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776431.420513:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491b800 x1862022710414080/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.420516:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776431.420518:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491b800 x1862022710414080/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.420522:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776431.420523:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776431.420524:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776431.420524:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776431.420525:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776431.420526:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776431.420528:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776431.420529:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776431.420531:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776431.420533:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491b800 x1862022710414080/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776431.420537:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710414080:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776431.420539:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776431.420541:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776431.420543:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918a80 x1862022710413952/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.420547:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776431.420548:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776431.420549:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775776431.420550:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776431.420551:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776431.420551:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776431.420553:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776431.420554:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4f400. 02000000:00000001:0.0:1775776431.420555:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776431.420556:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776431.420557:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918a80. 00000100:00000001:0.0:1775776431.420559:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776431.420560:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776431.420561:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776431.420562:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491b800 x1862022710414080/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776456 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776431.420566:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776431.420566:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776431.420567:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776431.420568:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776431.420569:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776431.420569:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776431.420570:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776431.420571:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4c000. 02000000:00000001:0.0:1775776431.420572:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776431.420572:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776431.420573:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491b800. 00000100:00000001:0.0:1775776431.420574:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776431.420575:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776431.420576:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776431.420578:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776431.420579:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776431.420580:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776431.420581:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776431.547245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776431.547252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776431.547256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776431.547260:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776431.547262:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776431.547264:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776431.547267:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776431.547269:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776431.547270:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776431.547272:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776431.547275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776431.547276:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776431.547277:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776431.547277:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776431.547278:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776431.547279:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776431.547280:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776431.547281:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776431.547329:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776431.547334:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776431.547337:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776432.443247:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776432.443253:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776432.443255:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776432.443260:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776432.443264:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776432.443267:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776432.443270:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776432.443272:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776432.443275:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776432.571246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776432.571252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776432.571258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776432.571260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776432.571263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776432.571264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776432.571352:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776432.571358:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776432.571362:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776433.467249:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776433.467255:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776433.467256:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776433.467260:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776433.467262:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776433.467265:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776433.467267:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776433.467268:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776433.467270:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776433.595220:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776433.595225:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776433.595229:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776433.595230:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776433.595232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776433.595233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776433.595319:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776433.595324:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776433.595327:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776434.491225:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776434.491232:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776434.491234:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776434.491239:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776434.491242:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776434.491244:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776434.491246:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776434.491248:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776434.491250:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776434.619245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776434.619249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776434.619255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776434.619256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776434.619259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776434.619260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776434.619323:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776434.619327:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776434.619331:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776435.323252:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307990278, at 4307990336 00000100:00000010:0.0:1775776435.323265:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3480. 00000100:00000040:1.0:1775776435.323291:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307990278, at 4307990336 00000100:00000010:1.0:1775776435.323304:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3780. 00000100:00000001:0.0:1775776435.515244:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776435.515248:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776435.515249:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776435.515259:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776435.515262:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776435.515265:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776435.515268:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776435.515270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776435.515271:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776435.643277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776435.643285:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776435.643287:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776435.643290:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776435.643294:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776435.643296:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776435.643298:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776435.643299:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776435.643301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776435.643302:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776435.643305:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776435.643306:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776435.643367:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776435.643371:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776435.643373:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776436.539233:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776436.539241:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776436.539243:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776436.539246:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775776436.539248:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776436.539253:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776436.539253:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776436.539256:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775776436.539258:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775776436.539259:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776436.539262:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776436.539263:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775776436.539266:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775776436.539270:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13319 00000100:00000001:0.0:1775776436.539271:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776436.539277:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776436.539281:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776436.539284:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776436.539286:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776436.539288:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776436.539291:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776436.539294:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776436.539298:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776436.539299:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776436.539302:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776436.539304:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776436.539306:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776436.539308:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776436.539312:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776436.539313:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776436.539315:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776436.539317:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776436.539320:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776436.539324:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776436.539326:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776436.539329:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776436.539332:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776436.539334:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776436.539336:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776436.539338:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776436.539347:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775776436.539349:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776436.539352:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776436.539354:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776436.539356:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776436.539357:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776436.539360:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776436.539366:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000100:00000001:3.0:1775776436.539371:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776436.539377:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710414208/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776436.539383:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776436.539386:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776436.539389:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710414208/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776436.539405:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776436.539408:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776436.539410:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776436.539412:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776436.539414:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776436.539416:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776436.539419:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13319 00000400:00000200:3.0:1775776436.539425:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776436.539428:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:0.0:1775776436.539430:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1775776436.539431:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775776436.539434:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776436.539436:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776436.539441:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000040:0.0:1775776436.539445:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71500 x1862022710414208/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776436.539445:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776436.539449:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776436.539453:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776436.539457:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776436.539459:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776436.539462:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775776436.539464:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1775776436.539465:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:0.0:1775776436.539468:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775776436.539468:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776436.539472:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:0.0:1775776436.539475:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1775776436.539475:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776436.539477:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775776436.539479:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1775776436.539479:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776436.539482:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:0.0:1775776436.539485:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776436.539485:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776436.539487:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776436.539490:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776436.539492:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:0.0:1775776436.539493:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710414208:192.168.201.115@tcp:250:kworker.0 02000000:00000001:3.0:1775776436.539497:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776436.539498:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776436.539501:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776436.539502:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1775776436.539502:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775776436.539505:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776436.539507:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:3.0:1775776436.539507:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776436.539510:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776436.539511:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776436.539514:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4400. 00000100:00000010:3.0:1775776436.539514:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 02000000:00000001:0.0:1775776436.539518:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776436.539519:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776436.539523:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776436.539524:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776436.539526:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776436.539527:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:0.0:1775776436.539528:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776436.539529:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776436.539535:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 02000000:00000010:3.0:1775776436.539535:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000100:00000001:3.0:1775776436.539550:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776436.539555:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00100000:3.0:1775776436.539555:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710414336/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776436.539563:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776436.539564:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775776436.539565:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710414208, portal 25 00000100:00000040:3.0:1775776436.539567:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710414336/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.539569:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776436.539572:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849009408 : -123823860542208 : ffff8f6202c71500) 00000100:00000001:3.0:1775776436.539574:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776436.539577:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776436.539581:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000040:0.0:1775776436.539582:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71500 x1862022710414208/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776461 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776436.539583:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776436.539585:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776436.539589:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776436.539593:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13319 00000100:00000001:0.0:1775776436.539598:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1775776436.539599:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:0.0:1775776436.539604:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000200:3.0:1775776436.539604:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776436.539606:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000010:0.0:1775776436.539609:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000040:3.0:1775776436.539609:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776436.539613:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:0.0:1775776436.539614:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710414208, offset 0 00000100:00000040:3.0:1775776436.539614:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776436.539618:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776436.539620:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776436.539621:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776436.539622:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776436.539623:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000400:00000200:0.0:1775776436.539626:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000020:00000001:3.0:1775776436.539626:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776436.539629:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776436.539632:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776436.539633:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776436.539636:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:0.0:1775776436.539637:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776436.539639:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776436.539642:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:0.0:1775776436.539646:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00080000:3.0:1775776436.539646:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776436.539648:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776436.539650:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776436.539652:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775776436.539653:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1775776436.539655:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776436.539657:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776436.539658:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1775776436.539659:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000008:00000001:3.0:1775776436.539660:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776436.539663:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776436.539666:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000010:3.0:1775776436.539666:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775776436.539669:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:0.0:1775776436.539672:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 02000000:00000001:3.0:1775776436.539672:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776436.539675:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776436.539676:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776436.539678:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:0.0:1775776436.539680:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453788:453788:256:4294967295] 192.168.201.115@tcp LPNI seq info [453713:453713:8:4294967295] 02000000:00000001:3.0:1775776436.539681:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776436.539684:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775776436.539688:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776436.539691:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00100000:3.0:1775776436.539694:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710414464/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776436.539699:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776436.539701:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776436.539708:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710414464/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776436.539709:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:3.0:1775776436.539720:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775776436.539722:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776436.539722:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776436.539727:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776436.539728:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:0.0:1775776436.539730:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000100:00000040:3.0:1775776436.539730:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776436.539736:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13329) 00000800:00000200:0.0:1775776436.539741:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776436.539745:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775776436.539750:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776436.539754:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775776436.539758:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776436.539761:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776436.539767:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776436.539771:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776436.539775:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776436.539778:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776436.539782:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776436.539786:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71500 x1862022710414208/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776461 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.539796:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776436.539799:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71500 x1862022710414208/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776461 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.539808:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776436.539809:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776436.539814:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776436.539817:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776436.539819:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4307995398 00000100:00000010:0.0:1775776436.539824:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776436.539830:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776436.539835:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4307995398 00000100:00000010:0.0:1775776436.539838:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3d80. 00000100:00000001:0.0:1775776436.539841:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776436.539844:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776436.539846:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776436.539848:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776436.539852:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776436/real 1775776436] req@ffff8f6202c71500 x1862022710414208/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776436.539861:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776436.539862:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776436.539864:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776436.539866:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776436.539868:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776436.539870:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776436.539873:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71500 x1862022710414208/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776436.539881:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71500 x1862022710414208/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.539888:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776436.539889:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776436.539892:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776436.539893:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776436.539896:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71500 x1862022710414208/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.539903:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776436.539907:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71500 x1862022710414208/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.539914:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776436.539916:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776436.539918:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776436.539919:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776436.539921:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776436.539923:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776436.539926:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776436.539929:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776436.539932:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776436.539936:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71500 x1862022710414208/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776436.539944:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710414208:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776436.539948:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776436.539949:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776436.539952:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71500 x1862022710414208/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.539958:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776436.539960:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776436.539962:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776436.539965:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776436.539966:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776436.539968:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776436.539970:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776436.539972:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36400. 02000000:00000001:0.0:1775776436.539975:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776436.539977:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776436.539978:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71500. 00000100:00000001:0.0:1775776436.539981:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776436.539982:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776436.539984:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776436.539990:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776436.539991:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776436.539993:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776436.539994:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776436.539996:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776436.539999:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70a80 x1862022710414336/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.540006:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776436.540008:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776436.540009:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776436.540011:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776436.540013:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776436.540016:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710414336:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776436.540019:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776436.540020:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776436.540022:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776436.540023:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776436.540026:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776436.540028:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776436.540031:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776436.540034:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776436.540037:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776436.540039:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710414336, portal 10 00000100:00000001:0.0:1775776436.540041:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776436.540042:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849006720 : -123823860544896 : ffff8f6202c70a80) 00000100:00000040:0.0:1775776436.540048:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70a80 x1862022710414336/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776461 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.540054:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776436.540057:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776436.540059:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776436.540061:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710414336, offset 0 00000400:00000200:0.0:1775776436.540065:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776436.540068:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776436.540072:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776436.540074:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776436.540077:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776436.540079:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776436.540081:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776436.540084:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453789:453789:256:4294967295] 192.168.201.115@tcp LPNI seq info [453714:453714:8:4294967295] 00000400:00000200:0.0:1775776436.540088:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776436.540094:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776436.540098:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776436.540101:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776436.540104:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776436.540106:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775776436.540110:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776436.540112:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775776436.540115:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776436.540117:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776436.540119:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776436.540122:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776436.540124:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776436.540126:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776436.540128:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776436.540131:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70a80 x1862022710414336/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776461 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.540156:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776436.540159:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70a80 x1862022710414336/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776461 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.540164:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776436.540165:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776436.540169:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776436.540172:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776436.540175:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776436.540177:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307991564 00000100:00000001:0.0:1775776436.540179:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307995398 00000100:00000001:0.0:1775776436.540182:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307991564 00000100:00000001:0.0:1775776436.540184:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307995398 00000100:00000001:0.0:1775776436.540185:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776436.540187:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776436.540189:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776436.540192:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776436/real 1775776436] req@ffff8f6202c70a80 x1862022710414336/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776436.540199:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776436.540200:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776436.540202:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776436.540204:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776436.540205:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776436.540207:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776436.540209:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70a80 x1862022710414336/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776436.540217:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70a80 x1862022710414336/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.540223:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776436.540224:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776436.540226:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776436.540227:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776436.540230:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70a80 x1862022710414336/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.540236:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776436.540239:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70a80 x1862022710414336/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.540245:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776436.540247:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776436.540248:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776436.540249:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776436.540250:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776436.540255:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776436.540257:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776436.540259:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776436.540261:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776436.540265:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70a80 x1862022710414336/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776436.540272:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710414336:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776436.540276:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776436.540278:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70700 x1862022710414464/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.540284:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776436.540285:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776436.540287:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776436.540289:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776436.540291:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776436.540294:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710414464:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776436.540297:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776436.540298:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776436.540299:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776436.540301:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776436.540305:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775776436.540306:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776436.540309:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776436.540312:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776436.540314:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776436.540317:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710414464, portal 10 00000100:00000001:0.0:1775776436.540319:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776436.540320:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849005824 : -123823860545792 : ffff8f6202c70700) 00000100:00000040:0.0:1775776436.540325:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70700 x1862022710414464/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776461 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.540331:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776436.540333:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776436.540335:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776436.540338:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710414464, offset 0 00000400:00000200:0.0:1775776436.540341:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776436.540345:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776436.540348:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776436.540351:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776436.540353:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776436.540355:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776436.540357:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776436.540360:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453790:453790:256:4294967295] 192.168.201.115@tcp LPNI seq info [453715:453715:8:4294967295] 00000400:00000200:0.0:1775776436.540364:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776436.540369:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776436.540374:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776436.540377:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776436.540381:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776436.540383:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775776436.540387:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776436.540389:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775776436.540393:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776436.540394:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776436.540406:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776436.540413:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776436.540416:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776436.540418:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776436.540421:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776436.540424:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70700 x1862022710414464/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776461 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.540431:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776436.540434:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70700 x1862022710414464/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776461 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.540439:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776436.540440:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776436.540443:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776436.540445:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776436.540448:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776436.540450:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307991564 00000100:00000001:0.0:1775776436.540453:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4307995398 00000100:00000001:0.0:1775776436.540455:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307991564 00000100:00000001:0.0:1775776436.540456:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4307995398 00000100:00000001:0.0:1775776436.540458:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776436.540460:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776436.540462:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776436.540466:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776436/real 1775776436] req@ffff8f6202c70700 x1862022710414464/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776436.540473:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776436.540474:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776436.540476:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776436.540478:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776436.540479:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776436.540481:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776436.540484:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70700 x1862022710414464/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776436.540492:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70700 x1862022710414464/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.540499:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776436.540501:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776436.540502:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776436.540504:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776436.540506:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70700 x1862022710414464/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.540512:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776436.540515:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70700 x1862022710414464/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.540522:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776436.540524:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776436.540525:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776436.540526:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776436.540528:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776436.540530:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776436.540532:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776436.540535:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776436.540548:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776436.540553:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70700 x1862022710414464/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776436.540561:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710414464:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776436.540564:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776436.540566:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776436.540569:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70a80 x1862022710414336/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.540575:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776436.540577:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776436.540578:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:0.0:1775776436.540581:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776436.540582:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776436.540583:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776436.540585:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776436.540587:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36000. 02000000:00000001:0.0:1775776436.540589:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776436.540590:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776436.540591:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70a80. 00000100:00000001:0.0:1775776436.540594:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776436.540595:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776436.540596:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776436.540599:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70700 x1862022710414464/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776461 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776436.540605:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776436.540606:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776436.540608:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:0.0:1775776436.540610:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776436.540611:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776436.540612:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776436.540613:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776436.540615:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34c00. 02000000:00000001:0.0:1775776436.540617:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776436.540618:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776436.540619:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70700. 00000100:00000001:0.0:1775776436.540621:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776436.540622:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776436.540624:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776436.540627:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776436.540628:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776436.540630:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776436.540631:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776436.667251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776436.667255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776436.667259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776436.667259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776436.667262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776436.667263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776436.667321:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776436.667325:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776436.667328:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776437.563239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776437.563245:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776437.563247:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776437.563252:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776437.563257:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776437.563260:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776437.563263:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776437.563265:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776437.563267:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776437.691282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776437.691287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776437.691293:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776437.691295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776437.691298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776437.691299:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776437.691359:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776437.691362:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776437.691365:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776438.587216:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776438.587223:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776438.587225:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776438.587230:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776438.587234:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776438.587237:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776438.587240:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776438.587242:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776438.587245:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776438.715235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776438.715238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776438.715242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776438.715243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776438.715245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776438.715246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776438.715302:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776438.715306:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776438.715308:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776439.611233:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776439.611239:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776439.611240:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776439.611244:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776439.611246:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776439.611249:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776439.611251:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776439.611253:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776439.611256:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776439.739230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776439.739236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776439.739242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776439.739243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776439.739247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776439.739248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776439.739307:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776439.739311:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776439.739314:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776440.443236:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4307995398, at 4307995456 00000100:00000040:0.0:1775776440.443243:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4307995398, at 4307995456 00000100:00000010:1.0:1775776440.443250:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3d80. 00000100:00000010:0.0:1775776440.443251:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776440.635243:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776440.635249:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776440.635251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776440.635257:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776440.635260:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776440.635264:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776440.635273:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776440.635276:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776440.635279:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776440.763281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776440.763288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776440.763294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776440.763295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776440.763299:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776440.763301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776440.763357:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776440.763363:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776440.763366:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776441.659256:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776441.659265:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776441.659267:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776441.659273:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776441.659277:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776441.659281:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776441.659284:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776441.659287:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776441.659290:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776441.659316:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776441.659322:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776441.659325:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776441.659328:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776441.659332:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776441.659341:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13324 00000400:00000200:0.0:1775776441.659351:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776441.659359:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776441.659362:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776441.659366:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776441.659369:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776441.659371:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776441.659374:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776441.659378:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776441.659380:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776441.659384:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776441.659386:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776441.659389:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776441.659400:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776441.659406:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776441.659408:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776441.659410:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776441.659412:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776441.659416:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776441.659419:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776441.659422:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776441.659426:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776441.659429:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776441.659432:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776441.659435:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776441.659437:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776441.659447:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063b80. 00000020:00000040:0.0:1775776441.659478:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776441.659482:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776441.659484:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776441.659487:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776441.659490:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776441.659495:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776441.659504:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6400. 00000100:00000001:0.0:1775776441.659512:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776441.659522:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063b80 x1862022710414592/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776441.659533:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776441.659538:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776441.659544:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ee9007b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063b80 x1862022710414592/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776441.659571:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776441.659575:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776441.659578:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776441.659579:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776441.659581:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776441.659583:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776441.659586:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13324 00000400:00000200:0.0:1775776441.659591:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776441.659595:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776441.659597:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776441.659601:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776441.659601:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776441.659605:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776441.659605:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776441.659608:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776441.659610:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776441.659615:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:1.0:1775776441.659616:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e063b80 x1862022710414592/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775776441.659619:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776441.659622:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776441.659626:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775776441.659627:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:0.0:1775776441.659629:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:1.0:1775776441.659629:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776441.659634:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776441.659634:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776441.659637:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775776441.659638:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776441.659639:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776441.659646:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00100000:1.0:1775776441.659646:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e063b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710414592:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776441.659652:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776441.659653:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1775776441.659657:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776441.659658:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1775776441.659660:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776441.659661:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775776441.659664:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:1.0:1775776441.659665:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1775776441.659669:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000010:1.0:1775776441.659671:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4dc00. 00000100:00000001:0.0:1775776441.659672:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776441.659673:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776441.659677:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776441.659681:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:0.0:1775776441.659684:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1775776441.659684:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 02000000:00000001:0.0:1775776441.659688:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776441.659690:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000001:0.0:1775776441.659691:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776441.659697:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000200:1.0:1775776441.659699:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710414592, portal 25 00000008:00000001:0.0:1775776441.659702:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776441.659703:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776441.659706:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574568320 : -123823134983296 : ffff8f622e063b80) 00000100:00000001:0.0:1775776441.659707:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776441.659711:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e063b80 x1862022710414592/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776466 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775776441.659712:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061f80. 00000020:00000040:0.0:1775776441.659717:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000001:1.0:1775776441.659720:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1775776441.659723:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1775776441.659724:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 02000000:00000001:0.0:1775776441.659726:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776441.659728:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000001:0.0:1775776441.659730:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:1.0:1775776441.659731:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710414592, offset 0 02000000:00000001:0.0:1775776441.659734:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776441.659738:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776441.659738:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 02000000:00000010:0.0:1775776441.659744:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4400. 00000400:00000200:1.0:1775776441.659747:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776441.659748:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776441.659754:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00100000:0.0:1775776441.659755:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061f80 x1862022710414720/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776441.659760:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776441.659766:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:0.0:1775776441.659767:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776441.659771:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776441.659771:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776441.659775:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000040:0.0:1775776441.659780:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000029d3794] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061f80 x1862022710414720/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776441.659780:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453791:453791:256:4294967295] 192.168.201.115@tcp LPNI seq info [453716:453716:8:4294967295] 00000400:00000200:1.0:1775776441.659789:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776441.659798:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776441.659802:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00080000:0.0:1775776441.659803:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776441.659809:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000200:1.0:1775776441.659810:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:0.0:1775776441.659813:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776441.659816:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000010:1.0:1775776441.659816:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000100:00000001:0.0:1775776441.659818:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776441.659823:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13324 00000800:00000200:1.0:1775776441.659826:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000400:00000200:0.0:1775776441.659830:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000200:1.0:1775776441.659830:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000400:00000200:0.0:1775776441.659835:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000800:00000200:1.1:1775776441.659836:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00080000:0.0:1775776441.659841:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000800:00000200:1.1:1775776441.659841:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000100:00000040:0.0:1775776441.659848:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000800:00000100:1.0:1775776441.659849:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000001:0.0:1775776441.659854:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:1.0:1775776441.659854:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000100:00000040:0.0:1775776441.659859:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776441.659861:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776441.659861:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:0.0:1775776441.659864:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776441.659867:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000400:00000200:1.0:1775776441.659868:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000020:00000001:0.0:1775776441.659871:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776441.659873:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000400:00000200:1.0:1775776441.659874:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000020:00000001:0.0:1775776441.659877:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000010:1.0:1775776441.659877:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000020:00000001:0.0:1775776441.659882:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000001:1.0:1775776441.659885:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:0.0:1775776441.659888:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000200:1.0:1775776441.659890:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e063b80 x1862022710414592/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776466 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776441.659892:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776441.659898:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776441.659901:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:1.0:1775776441.659901:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776441.659905:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e063b80 x1862022710414592/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776466 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775776441.659906:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775776441.659914:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776441.659915:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776441.659918:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776441.659919:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776441.659922:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776441.659924:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776441.659926:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775776441.659930:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:0.0:1775776441.659931:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776441.659933:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308000518 00000100:00000001:0.0:1775776441.659935:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776441.659938:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000010:1.0:1775776441.659939:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2ba0. 00000008:00000001:0.0:1775776441.659943:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776441.659946:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776441.659948:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000010:0.0:1775776441.659950:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061180. 00000020:00000040:0.0:1775776441.659953:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776441.659957:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776441.659959:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776441.659960:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308000518 00000100:00000001:0.0:1775776441.659961:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776441.659964:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776441.659966:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776441.659966:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e27e0. 00000100:00000001:1.0:1775776441.659969:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 02000000:00000010:0.0:1775776441.659973:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000001:1.0:1775776441.659973:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776441.659977:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776441.659980:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776441.659980:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776441.659985:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776441/real 1775776441] req@ffff8f622e063b80 x1862022710414592/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776441.659990:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061180 x1862022710414848/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1775776441.659995:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776441.659998:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776441.660000:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000100:00000001:0.0:1775776441.660004:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775776441.660005:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000100:00000001:0.0:1775776441.660008:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776441.660008:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776441.660013:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:0.0:1775776441.660015:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e23859] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061180 x1862022710414848/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776441.660019:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e063b80 x1862022710414592/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776441.660037:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776441.660041:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1775776441.660042:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e063b80 x1862022710414592/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776441.660050:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776441.660056:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776441.660056:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776441.660059:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776441.660063:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776441.660064:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776441.660066:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776441.660070:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e063b80 x1862022710414592/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776441.660072:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13334) 00000100:00000001:1.0:1775776441.660078:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776441.660082:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e063b80 x1862022710414592/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776441.660087:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776441.660089:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776441.660090:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776441.660090:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776441.660092:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776441.660094:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776441.660095:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776441.660099:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776441.660101:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776441.660104:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e063b80 x1862022710414592/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776441.660108:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e063b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710414592:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776441.660113:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776441.660114:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776441.660116:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e063b80 x1862022710414592/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776441.660120:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776441.660122:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776441.660124:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4dc00. 02000000:00000001:1.0:1775776441.660126:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776441.660127:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776441.660128:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776441.660129:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776441.660131:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:1.0:1775776441.660153:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776441.660154:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776441.660156:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e063b80. 00000100:00000001:1.0:1775776441.660158:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776441.660159:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776441.660160:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776441.660166:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776441.660167:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776441.660169:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776441.660170:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776441.660171:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776441.660173:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061f80 x1862022710414720/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776441.660177:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776441.660178:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776441.660180:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776441.660181:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776441.660182:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776441.660184:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710414720:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776441.660186:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776441.660187:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776441.660188:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776441.660190:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776441.660192:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4dc00. 02000000:00000001:1.0:1775776441.660193:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776441.660196:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776441.660197:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776441.660200:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776441.660203:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710414720, portal 10 00000100:00000001:1.0:1775776441.660204:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776441.660206:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574561152 : -123823134990464 : ffff8f622e061f80) 00000100:00000040:1.0:1775776441.660213:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061f80 x1862022710414720/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776466 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776441.660219:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776441.660221:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776441.660223:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776441.660226:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710414720, offset 0 00000400:00000200:1.0:1775776441.660230:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776441.660233:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776441.660237:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776441.660240:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776441.660242:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776441.660245:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776441.660247:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776441.660250:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453792:453792:256:4294967295] 192.168.201.115@tcp LPNI seq info [453717:453717:8:4294967295] 00000400:00000200:1.0:1775776441.660254:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776441.660259:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776441.660262:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776441.660265:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776441.660267:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776441.660269:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775776441.660272:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776441.660273:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1775776441.660275:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776441.660276:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776441.660279:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776441.660281:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776441.660283:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776441.660284:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776441.660286:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776441.660288:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061f80 x1862022710414720/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776466 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776441.660293:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776441.660295:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061f80 x1862022710414720/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776466 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776441.660299:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776441.660300:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776441.660302:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776441.660305:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776441.660307:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776441.660308:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307996684 00000100:00000001:1.0:1775776441.660310:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308000518 00000100:00000001:1.0:1775776441.660312:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307996684 00000100:00000001:1.0:1775776441.660313:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308000518 00000100:00000001:1.0:1775776441.660314:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776441.660315:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776441.660317:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776441.660318:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776441/real 1775776441] req@ffff8f622e061f80 x1862022710414720/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776441.660324:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776441.660326:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776441.660327:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776441.660329:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776441.660331:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776441.660333:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776441.660336:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061f80 x1862022710414720/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776441.660344:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061f80 x1862022710414720/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776441.660350:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776441.660351:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776441.660353:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776441.660355:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776441.660358:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061f80 x1862022710414720/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776441.660365:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776441.660369:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061f80 x1862022710414720/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776441.660375:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776441.660377:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776441.660378:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776441.660379:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776441.660380:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776441.660384:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776441.660386:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776441.660388:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776441.660390:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776441.660392:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061f80 x1862022710414720/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776441.660396:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710414720:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776441.660399:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776441.660400:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061180 x1862022710414848/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776441.660409:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776441.660410:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776441.660411:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776441.660412:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776441.660413:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776441.660415:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710414848:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776441.660417:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776441.660418:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776441.660419:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776441.660420:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776441.660423:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776441.660425:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776441.660427:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776441.660430:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776441.660433:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776441.660435:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710414848, portal 10 00000100:00000001:1.0:1775776441.660437:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776441.660439:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574557568 : -123823134994048 : ffff8f622e061180) 00000100:00000040:1.0:1775776441.660444:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061180 x1862022710414848/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776466 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776441.660463:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776441.660465:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776441.660467:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000200:1.0:1775776441.660468:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710414848, offset 0 00000400:00000200:1.0:1775776441.660471:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776441.660474:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776441.660476:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776441.660478:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776441.660479:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776441.660481:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776441.660482:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776441.660485:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453793:453793:256:4294967295] 192.168.201.115@tcp LPNI seq info [453718:453718:8:4294967295] 00000400:00000200:1.0:1775776441.660487:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776441.660492:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776441.660494:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776441.660496:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000800:00000200:1.0:1775776441.660499:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776441.660501:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775776441.660503:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776441.660505:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1775776441.660507:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776441.660509:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:1.0:1775776441.660510:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776441.660513:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776441.660514:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:1.0:1775776441.660515:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:1.0:1775776441.660517:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776441.660519:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061180 x1862022710414848/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776466 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776441.660524:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776441.660526:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061180 x1862022710414848/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776466 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776441.660530:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776441.660531:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776441.660533:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776441.660534:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776441.660537:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776441.660538:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4307996684 00000100:00000001:1.0:1775776441.660540:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308000518 00000100:00000001:1.0:1775776441.660541:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4307996684 00000100:00000001:1.0:1775776441.660542:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308000518 00000100:00000001:1.0:1775776441.660544:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776441.660546:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776441.660547:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776441.660549:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776441/real 1775776441] req@ffff8f622e061180 x1862022710414848/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776441.660553:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776441.660554:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776441.660555:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776441.660556:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776441.660557:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776441.660558:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776441.660560:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061180 x1862022710414848/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776441.660565:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061180 x1862022710414848/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776441.660568:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776441.660569:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776441.660570:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776441.660571:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776441.660572:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061180 x1862022710414848/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776441.660576:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776441.660578:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061180 x1862022710414848/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776441.660581:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776441.660582:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776441.660583:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776441.660584:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776441.660585:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776441.660586:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776441.660587:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776441.660589:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776441.660590:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776441.660593:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061180 x1862022710414848/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776441.660597:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710414848:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776441.660599:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776441.660600:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776441.660601:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061f80 x1862022710414720/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776441.660604:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776441.660605:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776441.660606:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4dc00. 02000000:00000001:1.0:1775776441.660608:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776441.660608:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776441.660609:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776441.660610:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776441.660611:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:1.0:1775776441.660613:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776441.660614:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776441.660615:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061f80. 00000100:00000001:1.0:1775776441.660616:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776441.660617:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776441.660618:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776441.660620:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061180 x1862022710414848/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776466 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776441.660624:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776441.660624:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776441.660625:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776441.660626:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776441.660627:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776441.660627:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776441.660628:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776441.660629:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:1.0:1775776441.660631:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776441.660631:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776441.660632:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061180. 00000100:00000001:1.0:1775776441.660633:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776441.660634:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776441.660635:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776441.660637:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776441.660638:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776441.660639:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776441.660640:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776441.787296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776441.787304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776441.787310:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776441.787314:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776441.787317:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776441.787321:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776441.787324:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776441.787327:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776441.787330:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776441.787332:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776441.787336:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776441.787338:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776441.787339:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776441.787340:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776441.787342:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776441.787343:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776441.787345:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776441.787347:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776441.787406:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776441.787411:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776441.787414:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776442.683211:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776442.683221:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776442.683223:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776442.683231:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776442.683236:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776442.683240:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776442.683243:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776442.683246:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776442.683248:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776442.811242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776442.811248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776442.811254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776442.811255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776442.811258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776442.811260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776442.811309:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776442.811312:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776442.811315:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776443.707238:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776443.707244:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776443.707245:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776443.707249:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776443.707252:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776443.707256:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776443.707258:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776443.707261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776443.707263:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776443.835220:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776443.835227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776443.835232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776443.835233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776443.835237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776443.835238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776443.835295:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776443.835298:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776443.835300:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776444.731233:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776444.731238:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776444.731239:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776444.731244:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776444.731247:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776444.731250:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776444.731252:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776444.731253:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776444.731255:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776444.859242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776444.859245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776444.859249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776444.859250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776444.859252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776444.859253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776444.859312:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776444.859316:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776444.859319:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776445.563232:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308000518, at 4308000576 00000100:00000010:1.0:1775776445.563246:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e27e0. 00000100:00000040:0.0:1775776445.563250:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308000518, at 4308000576 00000100:00000010:0.0:1775776445.563255:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ba0. 00000100:00000001:0.0:1775776445.755229:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776445.755235:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776445.755237:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776445.755241:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776445.755243:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776445.755245:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776445.755247:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776445.755249:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776445.755251:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776445.883268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776445.883276:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776445.883278:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776445.883281:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776445.883286:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776445.883289:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776445.883291:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776445.883293:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776445.883297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776445.883298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776445.883301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776445.883303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776445.883361:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776445.883365:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776445.883368:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776446.779261:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775776446.779261:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776446.779268:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776446.779270:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776446.779270:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776446.779274:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1775776446.779275:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:3.0:1775776446.779278:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776446.779279:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776446.779283:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776446.779284:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775776446.779285:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776446.779288:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776446.779291:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776446.779292:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13329 00000400:00000200:3.0:1775776446.779301:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776446.779307:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776446.779310:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776446.779314:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776446.779317:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776446.779320:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776446.779322:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776446.779326:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776446.779327:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776446.779330:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776446.779332:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776446.779334:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776446.779336:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776446.779339:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776446.779341:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776446.779343:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776446.779345:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776446.779348:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776446.779351:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776446.779353:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776446.779356:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776446.779371:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776446.779374:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776446.779377:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776446.779379:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776446.779388:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775776446.779392:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776446.779396:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776446.779398:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776446.779400:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776446.779402:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776446.779407:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776446.779414:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000100:00000001:3.0:1775776446.779420:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776446.779429:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710414976/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776446.779439:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776446.779443:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776446.779447:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710414976/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776446.779471:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776446.779474:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776446.779477:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776446.779478:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776446.779480:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776446.779482:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776446.779485:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13329 00000400:00000200:3.0:1775776446.779489:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776446.779492:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776446.779495:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776446.779499:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776446.779501:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776446.779504:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776446.779507:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776446.779507:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:1.0:1775776446.779510:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1775776446.779511:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775776446.779513:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1775776446.779514:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776446.779517:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776446.779520:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000040:1.0:1775776446.779521:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71180 x1862022710414976/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775776446.779525:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776446.779532:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776446.779533:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776446.779536:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1775776446.779538:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776446.779540:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1775776446.779544:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776446.779545:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776446.779548:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1775776446.779553:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710414976:192.168.201.115@tcp:250:kworker.0 00000100:00000040:3.0:1775776446.779554:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776446.779558:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:1.0:1775776446.779561:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1775776446.779563:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:1.0:1775776446.779566:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1775776446.779567:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000001:1.0:1775776446.779570:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776446.779571:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776446.779575:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776446.779576:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1775776446.779581:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:1.0:1775776446.779582:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:3.0:1775776446.779584:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776446.779587:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776446.779588:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776446.779593:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1775776446.779596:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000008:00000001:3.0:1775776446.779599:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776446.779600:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000001:3.0:1775776446.779602:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776446.779606:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000010:3.0:1775776446.779607:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775776446.779611:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000200:1.0:1775776446.779613:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710414976, portal 25 02000000:00000001:3.0:1775776446.779614:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1775776446.779618:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1775776446.779618:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776446.779621:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849008512 : -123823860543104 : ffff8f6202c71180) 00000100:00000001:3.0:1775776446.779622:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776446.779624:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776446.779627:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776446.779631:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71180 x1862022710414976/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776471 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1775776446.779633:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775776446.779638:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776446.779646:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1775776446.779647:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710415104/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1775776446.779653:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776446.779658:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f36e8. 00000100:00000001:3.0:1775776446.779659:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776446.779663:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776446.779664:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710414976, offset 0 00000100:00000040:3.0:1775776446.779667:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710415104/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776446.779676:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776446.779676:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776446.779686:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00080000:3.0:1775776446.779686:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776446.779690:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776446.779691:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776446.779692:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775776446.779695:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:3.0:1775776446.779696:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776446.779700:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13329 00000400:00000200:1.0:1775776446.779702:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1775776446.779705:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776446.779710:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000400:00000200:1.0:1775776446.779711:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00080000:3.0:1775776446.779713:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1775776446.779719:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000040:3.0:1775776446.779719:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776446.779722:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776446.779724:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776446.779725:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776446.779727:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776446.779727:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776446.779728:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776446.779730:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776446.779731:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776446.779733:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776446.779735:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000040:3.0:1775776446.779737:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776446.779739:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776446.779743:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776446.779745:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453794:453794:256:4294967295] 192.168.201.115@tcp LPNI seq info [453719:453719:8:4294967295] 00000100:00000001:3.0:1775776446.779746:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776446.779748:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776446.779752:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776446.779755:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776446.779756:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776446.779757:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776446.779759:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776446.779761:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776446.779763:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776446.779765:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776446.779768:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776446.779771:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776446.779774:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000010:3.0:1775776446.779775:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775776446.779778:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776446.779781:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776446.779783:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776446.779784:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776446.779785:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776446.779787:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776446.779789:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776446.779792:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000100:00000001:3.0:1775776446.779795:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775776446.779798:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230100. 00000100:00100000:3.0:1775776446.779798:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710415232/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776446.779805:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776446.779807:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776446.779810:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776446.779813:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710415232/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1775776446.779821:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775776446.779823:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776446.779827:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776446.779829:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776446.779831:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776446.779833:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776446.779835:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13339) 00000100:00000001:1.0:1775776446.779841:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776446.779850:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776446.779852:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308005638 00000100:00000010:1.0:1775776446.779858:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e27e0. 00000100:00000001:1.0:1775776446.779864:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776446.779872:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308005638 00000100:00000010:1.0:1775776446.779875:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2e40. 00000100:00000001:1.0:1775776446.779879:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776446.779882:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776446.779885:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776446.779887:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776446.779890:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776446.779895:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776446.779897:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775776446.779899:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776446.779901:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776446.779902:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776446.779906:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72a00 x1862022710415104/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776446.779914:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776446.779915:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776446.779917:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776446.779919:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776446.779921:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776446.779924:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710415104:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776446.779927:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776446.779929:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776446.779930:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776446.779932:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776446.779935:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4dc00. 02000000:00000001:1.0:1775776446.779937:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776446.779940:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776446.779943:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fe000. 00000400:00000010:1.0:1775776446.779948:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776446.779951:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710415104, portal 10 00000100:00000001:1.0:1775776446.779952:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776446.779954:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849014784 : -123823860536832 : ffff8f6202c72a00) 00000100:00000040:1.0:1775776446.779960:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72a00 x1862022710415104/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776471 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776446.779967:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776446.779969:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776446.779972:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f37f8. 00000100:00000200:1.0:1775776446.779974:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710415104, offset 0 00000400:00000200:1.0:1775776446.779978:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776446.779982:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776446.779987:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776446.779989:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776446.779992:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776446.779996:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776446.779998:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776446.780002:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453795:453795:255:4294967295] 192.168.201.115@tcp LPNI seq info [453720:453720:7:4294967295] 00000400:00000200:1.0:1775776446.780006:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776446.780012:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776446.780016:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776446.780019:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230000. 00000800:00000200:1.0:1775776446.780023:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776446.780027:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:1.0:1775776446.780029:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776446.780033:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776446.780043:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776446.780045:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308001803 00000100:00000001:1.0:1775776446.780048:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308005638 00000100:00000001:1.0:1775776446.780050:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308001803 00000100:00000001:1.0:1775776446.780052:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308005638 00000100:00000001:1.0:1775776446.780054:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776446.780056:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776446.780058:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776446.780062:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72300 x1862022710415232/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776446.780075:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776446.780076:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776446.780078:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776446.780080:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776446.780082:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776446.780085:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710415232:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776446.780088:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776446.780090:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776446.780091:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776446.780093:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776446.780096:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776446.780098:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776446.780101:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776446.780103:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776446.780106:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776446.780109:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710415232, portal 10 00000100:00000001:1.0:1775776446.780111:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776446.780113:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849012992 : -123823860538624 : ffff8f6202c72300) 00000100:00000040:1.0:1775776446.780118:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72300 x1862022710415232/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776471 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776446.780125:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776446.780127:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776446.780129:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3c38. 00000100:00000200:1.0:1775776446.780131:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710415232, offset 0 00000400:00000200:1.0:1775776446.780161:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776446.780165:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776446.780168:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776446.780171:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776446.780173:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776446.780175:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776446.780177:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776446.780181:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453796:453796:254:4294967295] 192.168.201.115@tcp LPNI seq info [453721:453721:6:4294967295] 00000400:00000200:1.0:1775776446.780185:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776446.780191:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776446.780195:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776446.780199:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230c00. 00000800:00000200:1.0:1775776446.780203:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776446.780207:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:1.0:1775776446.780209:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776446.780212:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776446.780215:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776446.780217:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308001804 00000100:00000001:1.0:1775776446.780219:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308005638 00000100:00000001:1.0:1775776446.780221:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308001804 00000100:00000001:1.0:1775776446.780224:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308005638 00000100:00000001:1.0:1775776446.780226:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776446.780228:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776446.780230:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776446.780232:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776446.780235:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776446.780236:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775776446.780238:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776446.780240:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776446.780241:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776446.780243:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776446.907247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776446.907253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776446.907259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776446.907261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776446.907265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776446.907266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776446.907317:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776446.907320:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776446.907322:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776447.803261:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776447.803266:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776447.803267:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776447.803271:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776447.803273:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776447.803276:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776447.803278:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776447.803279:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776447.803281:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776447.931263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776447.931268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776447.931274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776447.931275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776447.931279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776447.931280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776447.931335:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776447.931338:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776447.931360:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776448.827290:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776448.827297:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776448.827299:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776448.827304:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776448.827307:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776448.827311:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776448.827313:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776448.827316:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776448.827319:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776448.955229:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776448.955234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776448.955240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776448.955241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776448.955243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776448.955244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776448.955289:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776448.955292:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776448.955294:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776449.851361:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776449.851365:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776449.851366:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776449.851370:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776449.851372:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776449.851375:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776449.851377:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776449.851379:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776449.851380:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:3.0:1775776449.851519:0:2399:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:3.0:1775776449.851556:0:2399:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1775776449.851566:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776449.851570:0:2399:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:3.0:1775776449.851575:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776449.851579:0:2399:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:3.0:1775776449.851583:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775776449.851588:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230100. 00000400:00000200:3.0:1775776449.851593:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775776449.851599:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775776449.851605:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f36e8 00000400:00000010:3.0:1775776449.851608:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f36e8. 00000100:00000001:3.0:1775776449.851614:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776449.851623:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c71180 x1862022710414976/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776471 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776449.851657:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776449.851660:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71180 x1862022710414976/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776471 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776449.851667:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776449.851669:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775776449.851684:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775776449.851686:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230000. 00000400:00000200:3.0:1775776449.851689:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775776449.851692:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775776449.851695:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f37f8 00000400:00000010:3.0:1775776449.851697:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f37f8. 00000100:00000001:3.0:1775776449.851700:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776449.851703:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c72a00 x1862022710415104/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776471 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776449.851710:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776449.851712:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72a00 x1862022710415104/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776471 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776449.851718:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776449.851720:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775776449.851726:0:2399:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775776449.851728:0:2399:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230c00. 00000400:00000200:3.0:1775776449.851731:0:2399:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775776449.851733:0:2399:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775776449.851736:0:2399:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3c38 00000400:00000010:3.0:1775776449.851743:0:2399:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3c38. 00000100:00000001:3.0:1775776449.851746:0:2399:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776449.851750:0:2399:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c72300 x1862022710415232/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776471 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776449.851757:0:2399:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776449.851759:0:2399:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72300 x1862022710415232/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776471 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776449.851764:0:2399:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776449.851766:0:2399:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776449.851916:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776449.851919:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776449.851922:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776449.851930:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776446/real 1775776449] req@ffff8f6202c72a00 x1862022710415104/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776449.851947:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776449.851949:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776449.851953:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fe000. 00000400:00000200:1.0:1775776449.851957:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776449.851958:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:1.0:1775776449.851962:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776449.851965:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72a00 x1862022710415104/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776449.851973:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72a00 x1862022710415104/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776449.851979:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776449.851995:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776449.851998:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776449.851999:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776449.852003:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72a00 x1862022710415104/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776449.852009:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776449.852013:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72a00 x1862022710415104/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776449.852020:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776449.852022:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776449.852023:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776449.852031:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776449.852033:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776449.852039:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776449.852042:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776449.852056:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776449.852059:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776449.852064:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72a00 x1862022710415104/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776449.852071:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710415104:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776449.852075:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776449.852078:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776446/real 1775776449] req@ffff8f6202c72300 x1862022710415232/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776449.852084:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776449.852086:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776449.852087:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000400:00000200:1.0:1775776449.852090:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776449.852092:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776449.852094:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776449.852096:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72300 x1862022710415232/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776449.852104:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72300 x1862022710415232/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776449.852113:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776449.852115:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776449.852120:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776449.852122:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776449.852125:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72300 x1862022710415232/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776449.852130:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776449.852164:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72300 x1862022710415232/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776449.852170:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776449.852172:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776449.852174:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776449.852175:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776449.852177:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776449.852179:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776449.852181:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776449.852184:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776449.852187:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776449.852191:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72300 x1862022710415232/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776449.852197:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710415232:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776449.852201:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776449.852204:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776446/real 1775776449] req@ffff8f6202c71180 x1862022710414976/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776449.852210:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776449.852211:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776449.852213:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776449.852215:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776449.852218:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776449.852220:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776449.852224:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71180 x1862022710414976/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776449.852232:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71180 x1862022710414976/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776449.852237:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776449.852239:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776449.852240:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776449.852241:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776449.852245:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71180 x1862022710414976/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776449.852254:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776449.852257:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71180 x1862022710414976/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776449.852262:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776449.852264:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776449.852265:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776449.852266:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776449.852267:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776449.852268:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776449.852270:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776449.852273:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776449.852275:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776449.852279:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71180 x1862022710414976/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776449.852285:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710414976:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776449.852289:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776449.852291:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776449.852294:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72a00 x1862022710415104/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776449.852300:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776449.852303:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776449.852319:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4dc00. 02000000:00000001:1.0:1775776449.852323:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776449.852326:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776449.852327:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776449.852329:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776449.852332:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34c00. 02000000:00000001:1.0:1775776449.852335:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776449.852337:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776449.852339:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72a00. 00000100:00000001:1.0:1775776449.852344:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776449.852345:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776449.852346:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776449.852350:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72300 x1862022710415232/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776449.852357:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776449.852358:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776449.852359:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776449.852362:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776449.852363:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776449.852364:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776449.852366:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776449.852368:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36400. 02000000:00000001:1.0:1775776449.852370:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776449.852371:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776449.852380:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72300. 00000100:00000001:1.0:1775776449.852383:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776449.852384:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776449.852389:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776449.852392:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71180 x1862022710414976/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776471 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776449.852398:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776449.852399:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776449.852401:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776449.852403:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776449.852404:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776449.852405:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776449.852407:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776449.852408:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36000. 02000000:00000001:1.0:1775776449.852410:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776449.852411:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776449.852412:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71180. 00000100:00000001:1.0:1775776449.852414:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776449.852415:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776449.852418:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776449.852425:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776449.852426:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776449.852429:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776449.852430:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776449.979247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776449.979252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776449.979255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776449.979256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776449.979259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776449.979259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776449.979351:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776449.979354:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776449.979356:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776450.683226:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308005638, at 4308005696 00000800:00000200:0.0:1775776450.683238:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000100:00000010:1.0:1775776450.683239:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2e40. 00000800:00000200:0.0:1775776450.683244:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:0.0:1775776450.683247:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000100:00000040:0.0:1775776450.683273:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308005638, at 4308005696 00000100:00000010:0.0:1775776450.683278:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e27e0. 00000800:00000001:0.0:1775776450.683367:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000200:1.0:1775776450.683375:0:2405:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1775776450.683381:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775776450.683385:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776450.683388:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776450.875341:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776450.875347:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776450.875349:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776450.875355:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776450.875359:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776450.875362:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776450.875365:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776450.875368:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776450.875371:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776451.003229:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776451.003234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776451.003238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776451.003239:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776451.003241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776451.003242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776451.003306:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776451.003311:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776451.003315:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776451.899259:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776451.899299:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776451.899304:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776451.899308:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776451.899314:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776451.899323:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13334 00000400:00000200:3.0:1775776451.899337:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776451.899346:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776451.899350:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775776451.899352:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000040:3.0:1775776451.899356:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776451.899358:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776451.899360:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776451.899361:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776451.899365:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00100000:0.0:1775776451.899366:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776451.899370:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:0.0:1775776451.899371:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1775776451.899376:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:0.0:1775776451.899377:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1775776451.899378:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000100:00000001:0.0:1775776451.899383:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1775776451.899383:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775776451.899386:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1775776451.899387:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776451.899391:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000001:0.0:1775776451.899392:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775776451.899396:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776451.899399:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776451.899401:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776451.899402:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776451.899404:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776451.899406:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776451.899409:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776451.899411:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776451.899414:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776451.899418:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776451.899422:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776451.899424:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776451.899427:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776451.899435:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73b80. 00000020:00000040:3.0:1775776451.899440:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776451.899444:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776451.899446:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776451.899450:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776451.899453:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776451.899459:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776451.899470:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000001:3.0:1775776451.899479:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776451.899491:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73b80 x1862022710415360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776451.899501:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776451.899507:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776451.899512:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c52008f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73b80 x1862022710415360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776451.899544:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776451.899547:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776451.899564:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776451.899566:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776451.899568:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776451.899572:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776451.899575:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13334 00000400:00000200:3.0:1775776451.899581:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776451.899585:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776451.899587:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776451.899590:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1775776451.899591:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776451.899595:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776451.899598:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775776451.899599:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776451.899602:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776451.899603:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776451.899605:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776451.899607:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776451.899609:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776451.899611:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776451.899613:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776451.899615:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776451.899619:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776451.899622:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776451.899625:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776451.899626:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776451.899629:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000040:1.0:1775776451.899633:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73b80 x1862022710415360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775776451.899634:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776451.899639:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776451.899645:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776451.899651:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1775776451.899654:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776451.899659:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776451.899660:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776451.899664:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776451.899667:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776451.899671:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1775776451.899671:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775776451.899678:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775776451.899679:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776451.899684:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776451.899687:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710415360:192.168.201.115@tcp:250:kworker.0 00000100:00000010:3.0:1775776451.899693:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000100:00000001:1.0:1775776451.899694:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1775776451.899698:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776451.899702:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776451.899703:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776451.899707:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776451.899709:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776451.899709:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776451.899713:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776451.899716:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1775776451.899716:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776451.899725:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 02000000:00000010:1.0:1775776451.899726:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776451.899730:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776451.899730:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776451.899739:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710415488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776451.899743:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776451.899751:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000001:3.0:1775776451.899753:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776451.899757:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776451.899762:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710415488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1775776451.899767:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000001:3.0:1775776451.899770:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776451.899776:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776451.899780:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776451.899782:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776451.899784:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1775776451.899787:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710415360, portal 25 00000100:00000001:3.0:1775776451.899788:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776451.899793:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13334 00000100:00000001:1.0:1775776451.899796:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000400:00000200:3.0:1775776451.899801:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776451.899803:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849019264 : -123823860532352 : ffff8f6202c73b80) 00000400:00000200:3.0:1775776451.899807:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776451.899812:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776451.899816:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73b80 x1862022710415360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776476 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776451.899820:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776451.899825:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776451.899830:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776451.899834:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776451.899841:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775776451.899843:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:3.0:1775776451.899843:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776451.899847:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000040:1.0:1775776451.899851:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000020:00000040:3.0:1775776451.899851:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776451.899856:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000010:1.0:1775776451.899857:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000020:00000001:3.0:1775776451.899859:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000200:1.0:1775776451.899863:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710415360, offset 0 00000100:00000040:3.0:1775776451.899866:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776451.899870:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776451.899872:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776451.899877:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776451.899878:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000020:00000040:3.0:1775776451.899883:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776451.899888:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776451.899891:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776451.899893:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776451.899896:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776451.899898:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776451.899902:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776451.899904:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775776451.899907:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000008:00100000:3.0:1775776451.899908:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776451.899914:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776451.899915:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776451.899919:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000010:3.0:1775776451.899920:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775776451.899924:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:1.0:1775776451.899928:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 02000000:00000001:3.0:1775776451.899929:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776451.899932:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776451.899935:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776451.899936:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1775776451.899937:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 02000000:00000001:3.0:1775776451.899940:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776451.899944:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 02000000:00000010:3.0:1775776451.899950:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000400:00000200:1.0:1775776451.899951:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453797:453797:256:4294967295] 192.168.201.115@tcp LPNI seq info [453722:453722:8:4294967295] 00000100:00000001:3.0:1775776451.899959:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776451.899961:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00100000:3.0:1775776451.899968:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710415616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776451.899982:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775776451.899984:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:3.0:1775776451.899986:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776451.899995:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710415616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1775776451.900000:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776451.900010:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775776451.900011:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000100:00000040:3.0:1775776451.900014:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776451.900021:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.0:1775776451.900027:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776451.900027:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776451.900035:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.0:1775776451.900036:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000100:00000040:3.0:1775776451.900043:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13344) 00000800:00000200:1.1:1775776451.900047:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776451.900054:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775776451.900059:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776451.900063:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776451.900072:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776451.900078:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776451.900086:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776451.900090:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776451.900097:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776451.900102:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73b80 x1862022710415360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776476 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.900112:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776451.900115:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73b80 x1862022710415360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776476 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.900129:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776451.900167:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776451.900176:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776451.900182:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776451.900186:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308010759 00000100:00000010:1.0:1775776451.900192:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2e40. 00000100:00000001:1.0:1775776451.900201:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776451.900210:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308010759 00000100:00000010:1.0:1775776451.900214:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e28a0. 00000100:00000001:1.0:1775776451.900217:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776451.900221:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776451.900224:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776451.900227:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776451.900233:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776451/real 1775776451] req@ffff8f6202c73b80 x1862022710415360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776451.900246:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776451.900248:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776451.900251:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776451.900254:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776451.900257:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776451.900260:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776451.900265:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73b80 x1862022710415360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776451.900292:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73b80 x1862022710415360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.900299:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776451.900301:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776451.900304:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776451.900306:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776451.900310:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73b80 x1862022710415360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.900318:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776451.900324:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73b80 x1862022710415360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.900332:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776451.900335:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776451.900337:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776451.900338:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776451.900341:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776451.900344:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776451.900348:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776451.900352:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776451.900355:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776451.900360:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73b80 x1862022710415360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776451.900368:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710415360:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776451.900373:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776451.900375:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776451.900377:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73b80 x1862022710415360/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.900384:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776451.900387:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776451.900389:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776451.900393:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776451.900395:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776451.900397:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776451.900399:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776451.900403:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34800. 02000000:00000001:1.0:1775776451.900407:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776451.900410:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776451.900412:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73b80. 00000100:00000001:1.0:1775776451.900416:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776451.900417:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776451.900420:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776451.900430:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776451.900432:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776451.900434:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776451.900436:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776451.900438:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776451.900442:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71180 x1862022710415488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.900448:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776451.900450:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776451.900452:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776451.900454:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776451.900457:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776451.900460:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710415488:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776451.900464:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776451.900466:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776451.900467:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776451.900469:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776451.900473:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776451.900475:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776451.900479:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776451.900482:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776451.900486:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776451.900490:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710415488, portal 10 00000100:00000001:1.0:1775776451.900492:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776451.900493:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849008512 : -123823860543104 : ffff8f6202c71180) 00000100:00000040:1.0:1775776451.900500:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71180 x1862022710415488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776476 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.900508:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776451.900511:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776451.900514:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776451.900517:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710415488, offset 0 00000400:00000200:1.0:1775776451.900521:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776451.900527:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776451.900531:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776451.900534:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776451.900537:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776451.900540:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776451.900542:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776451.900545:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453798:453798:256:4294967295] 192.168.201.115@tcp LPNI seq info [453723:453723:8:4294967295] 00000400:00000200:1.0:1775776451.900549:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776451.900556:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776451.900561:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776451.900565:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776451.900570:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776451.900574:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775776451.900578:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776451.900581:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775776451.900585:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776451.900587:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776451.900590:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776451.900593:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776451.900597:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776451.900599:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776451.900602:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776451.900605:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71180 x1862022710415488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776476 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.900613:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776451.900616:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71180 x1862022710415488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776476 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.900624:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776451.900625:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776451.900630:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776451.900635:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776451.900638:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776451.900642:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308006924 00000100:00000001:1.0:1775776451.900645:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308010759 00000100:00000001:1.0:1775776451.900647:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308006924 00000100:00000001:1.0:1775776451.900650:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308010759 00000100:00000001:1.0:1775776451.900652:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776451.900654:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776451.900656:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776451.900660:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776451/real 1775776451] req@ffff8f6202c71180 x1862022710415488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776451.900669:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776451.900671:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776451.900672:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776451.900676:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776451.900677:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776451.900679:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776451.900682:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71180 x1862022710415488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776451.900691:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71180 x1862022710415488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.900699:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776451.900701:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776451.900703:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776451.900704:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776451.900708:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71180 x1862022710415488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.900714:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776451.900718:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71180 x1862022710415488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.900727:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776451.900730:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776451.900732:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776451.900734:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776451.900736:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776451.900744:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776451.900747:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776451.900751:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776451.900754:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776451.900759:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71180 x1862022710415488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776451.900767:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710415488:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776451.900772:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776451.900775:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72300 x1862022710415616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.900782:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776451.900783:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776451.900785:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776451.900787:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776451.900789:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776451.900792:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710415616:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776451.900795:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776451.900798:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776451.900800:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776451.900801:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776451.900806:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4dc00. 02000000:00000001:1.0:1775776451.900808:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776451.900812:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776451.900814:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776451.900819:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776451.900822:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710415616, portal 10 00000100:00000001:1.0:1775776451.900824:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776451.900825:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849012992 : -123823860538624 : ffff8f6202c72300) 00000100:00000040:1.0:1775776451.900831:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72300 x1862022710415616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776476 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.900837:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776451.900839:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776451.900841:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776451.900843:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710415616, offset 0 00000400:00000200:1.0:1775776451.900848:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776451.900852:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776451.900855:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776451.900859:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776451.900862:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776451.900865:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776451.900868:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776451.900871:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453799:453799:256:4294967295] 192.168.201.115@tcp LPNI seq info [453724:453724:8:4294967295] 00000400:00000200:1.0:1775776451.900876:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776451.900881:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776451.900886:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776451.900890:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776451.900894:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776451.900897:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775776451.900901:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776451.900903:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775776451.900907:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776451.900909:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776451.900912:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776451.900917:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776451.900920:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776451.900923:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776451.900926:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776451.900929:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72300 x1862022710415616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776476 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.900937:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776451.900940:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72300 x1862022710415616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776476 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.900947:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776451.900948:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776451.900951:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776451.900955:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776451.900958:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776451.900961:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308006924 00000100:00000001:1.0:1775776451.900964:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308010759 00000100:00000001:1.0:1775776451.900967:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308006924 00000100:00000001:1.0:1775776451.900969:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308010759 00000100:00000001:1.0:1775776451.900971:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776451.900973:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776451.900975:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776451.900979:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776451/real 1775776451] req@ffff8f6202c72300 x1862022710415616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776451.900988:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776451.900989:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776451.900991:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776451.900994:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776451.900996:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776451.900998:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776451.901001:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72300 x1862022710415616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776451.901010:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72300 x1862022710415616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.901018:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776451.901019:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776451.901021:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776451.901022:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776451.901025:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72300 x1862022710415616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.901032:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776451.901037:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72300 x1862022710415616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.901043:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776451.901053:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776451.901055:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776451.901057:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776451.901059:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776451.901060:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776451.901063:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776451.901067:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776451.901070:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776451.901075:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72300 x1862022710415616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776451.901085:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710415616:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776451.901089:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776451.901091:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776451.901095:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71180 x1862022710415488/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.901101:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776451.901104:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776451.901105:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776451.901108:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776451.901109:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776451.901110:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776451.901112:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776451.901114:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34400. 02000000:00000001:1.0:1775776451.901116:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776451.901117:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776451.901119:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71180. 00000100:00000001:1.0:1775776451.901122:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776451.901124:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776451.901125:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776451.901129:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72300 x1862022710415616/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776476 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776451.901173:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776451.901175:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776451.901177:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4dc00. 02000000:00000001:1.0:1775776451.901180:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776451.901181:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776451.901183:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776451.901185:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776451.901187:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37000. 02000000:00000001:1.0:1775776451.901189:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776451.901191:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776451.901193:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72300. 00000100:00000001:1.0:1775776451.901196:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776451.901197:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776451.901199:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776451.901205:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776451.901206:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776451.901208:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776451.901209:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776452.027313:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776452.027326:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776452.027334:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776452.027339:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776452.027342:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776452.027347:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776452.027350:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776452.027353:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776452.027354:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776452.027356:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776452.027360:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776452.027361:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776452.027362:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776452.027363:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776452.027364:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776452.027365:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776452.027367:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776452.027368:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776452.027428:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776452.027433:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776452.027437:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776452.923408:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776452.923414:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776452.923416:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776452.923421:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776452.923424:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776452.923428:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776452.923430:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776452.923433:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776452.923435:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776453.051295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776453.051301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776453.051306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776453.051307:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776453.051309:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776453.051310:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776453.051375:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776453.051380:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776453.051384:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776453.947264:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776453.947280:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776453.947286:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776453.947292:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776453.947296:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776453.947300:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776453.947303:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776453.947306:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776453.947309:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776454.075269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776454.075274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776454.075278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776454.075279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776454.075282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776454.075283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776454.075335:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776454.075339:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776454.075343:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776454.971271:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776454.971276:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776454.971277:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776454.971282:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776454.971285:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776454.971288:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776454.971290:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776454.971292:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776454.971294:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776455.099287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776455.099294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776455.099300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776455.099301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776455.099305:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776455.099306:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776455.099357:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776455.099360:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776455.099363:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776455.803260:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308010759, at 4308010816 00000100:00000040:0.0:1775776455.803269:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308010759, at 4308010816 00000100:00000010:1.0:1775776455.803276:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e28a0. 00000100:00000010:0.0:1775776455.803278:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2e40. 00000100:00000001:0.0:1775776455.995264:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776455.995270:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776455.995271:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776455.995276:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776455.995279:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776455.995282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776455.995291:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776455.995293:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776455.995295:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776456.123313:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776456.123320:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776456.123322:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776456.123324:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776456.123329:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776456.123330:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776456.123332:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776456.123334:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776456.123336:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776456.123337:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776456.123340:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776456.123341:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776456.123407:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776456.123411:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776456.123415:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776457.019255:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776457.019263:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776457.019266:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776457.019270:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776457.019276:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776457.019282:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775776457.019284:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13339 00000100:00000001:2.0:1775776457.019286:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776457.019290:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776457.019293:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776457.019298:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1775776457.019298:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:2.0:1775776457.019302:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776457.019306:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:3.0:1775776457.019308:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:2.0:1775776457.019309:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776457.019315:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776457.019316:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776457.019321:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776457.019325:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776457.019328:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776457.019329:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776457.019332:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776457.019334:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776457.019336:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776457.019338:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776457.019339:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776457.019341:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776457.019343:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776457.019344:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776457.019345:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776457.019347:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776457.019349:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776457.019351:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776457.019352:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776457.019355:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776457.019357:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776457.019359:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776457.019360:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776457.019362:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776457.019368:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775776457.019370:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776457.019373:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776457.019374:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776457.019376:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776457.019378:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776457.019382:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776457.019388:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775776457.019394:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776457.019402:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710415744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776457.019410:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776457.019413:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776457.019417:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710415744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776457.019433:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776457.019435:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776457.019437:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776457.019439:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776457.019440:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776457.019442:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776457.019444:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13339 00000400:00000200:3.0:1775776457.019447:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776457.019449:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776457.019450:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776457.019452:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776457.019454:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776457.019456:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776457.019457:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776457.019459:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776457.019460:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776457.019461:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776457.019463:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776457.019464:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776457.019466:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776457.019468:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776457.019470:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776457.019472:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776457.019473:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776457.019474:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776457.019478:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000001:1.0:1775776457.019479:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776457.019481:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776457.019483:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776457.019486:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776457.019488:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1775776457.019491:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72a00 x1862022710415744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775776457.019493:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776457.019498:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776457.019502:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776457.019507:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1775776457.019508:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775776457.019509:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775776457.019511:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776457.019514:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1775776457.019514:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776457.019517:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1775776457.019518:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 02000000:00000001:1.0:1775776457.019523:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776457.019523:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776457.019529:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1775776457.019530:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710415744:192.168.201.115@tcp:250:kworker.0 02000000:00000001:3.0:1775776457.019531:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776457.019535:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775776457.019537:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776457.019540:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776457.019541:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776457.019543:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1775776457.019545:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776457.019547:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776457.019549:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4dc00. 02000000:00000010:3.0:1775776457.019551:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 02000000:00000001:1.0:1775776457.019552:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776457.019557:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776457.019561:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00100000:3.0:1775776457.019564:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710415872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1775776457.019566:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000001:3.0:1775776457.019574:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1775776457.019576:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000001:3.0:1775776457.019576:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776457.019582:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710415872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776457.019585:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710415744, portal 25 00000100:00000001:1.0:1775776457.019590:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776457.019592:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776457.019593:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849014784 : -123823860536832 : ffff8f6202c72a00) 00000100:00080000:3.0:1775776457.019595:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776457.019598:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000040:1.0:1775776457.019601:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72a00 x1862022710415744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776457.019601:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776457.019603:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776457.019608:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776457.019613:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13339 00000100:00000001:1.0:1775776457.019616:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1775776457.019619:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:1.0:1775776457.019623:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000200:3.0:1775776457.019624:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000400:00000010:1.0:1775776457.019627:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00080000:3.0:1775776457.019627:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000200:1.0:1775776457.019632:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710415744, offset 0 00000100:00000040:3.0:1775776457.019632:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776457.019635:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776457.019637:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776457.019638:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776457.019641:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:1.0:1775776457.019644:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000020:00000040:3.0:1775776457.019644:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776457.019645:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776457.019647:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776457.019649:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776457.019650:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776457.019654:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775776457.019657:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776457.019659:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776457.019662:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000040:3.0:1775776457.019663:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776457.019667:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776457.019669:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:3.0:1775776457.019673:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776457.019676:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00080000:3.0:1775776457.019680:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000400:00000200:1.0:1775776457.019683:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1775776457.019686:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776457.019687:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 02000000:00000001:3.0:1775776457.019691:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776457.019695:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453800:453800:256:4294967295] 192.168.201.115@tcp LPNI seq info [453725:453725:8:4294967295] 02000000:00000001:3.0:1775776457.019696:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775776457.019701:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776457.019702:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776457.019706:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776457.019708:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1775776457.019713:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000008:00000001:3.0:1775776457.019714:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776457.019718:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776457.019720:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000010:3.0:1775776457.019725:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000800:00000010:1.0:1775776457.019726:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000020:00000040:3.0:1775776457.019730:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000800:00000200:1.0:1775776457.019736:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:3.0:1775776457.019737:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776457.019740:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776457.019741:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000100:00000001:3.0:1775776457.019743:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776457.019746:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:1.1:1775776457.019749:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:3.0:1775776457.019750:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776457.019754:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 02000000:00000010:3.0:1775776457.019757:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000800:00000100:1.0:1775776457.019760:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776457.019764:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000100:00000001:3.0:1775776457.019764:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776457.019770:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00100000:3.0:1775776457.019773:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710416000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776457.019777:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776457.019781:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776457.019783:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776457.019787:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776457.019794:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72a00 x1862022710415744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776457.019796:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776457.019801:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776457.019808:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776457.019811:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710416000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776457.019814:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72a00 x1862022710415744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776482 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.019844:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776457.019844:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776457.019846:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1775776457.019850:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776457.019852:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776457.019858:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1775776457.019859:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776457.019864:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308015878 00000100:00000040:3.0:1775776457.019865:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000010:1.0:1775776457.019870:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e28a0. 00000100:00000040:3.0:1775776457.019870:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776457.019876:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13349) 00000100:00000001:1.0:1775776457.019878:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776457.019885:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308015878 00000100:00000010:1.0:1775776457.019887:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2a20. 00000100:00000001:1.0:1775776457.019890:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776457.019894:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776457.019896:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776457.019897:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776457.019900:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776457/real 1775776457] req@ffff8f6202c72a00 x1862022710415744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776457.019907:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776457.019908:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776457.019910:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776457.019912:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776457.019913:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776457.019915:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776457.019917:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72a00 x1862022710415744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776457.019923:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72a00 x1862022710415744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.019927:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776457.019928:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776457.019930:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776457.019931:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776457.019933:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72a00 x1862022710415744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.019938:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776457.019940:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72a00 x1862022710415744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.019945:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776457.019947:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776457.019948:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776457.019949:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776457.019951:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776457.019953:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776457.019955:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776457.019957:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776457.019959:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776457.019962:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72a00 x1862022710415744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776457.019967:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710415744:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776457.019970:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776457.019971:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776457.019973:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72a00 x1862022710415744/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.019978:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776457.019979:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776457.019981:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4dc00. 02000000:00000001:1.0:1775776457.019983:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776457.019984:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776457.019985:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776457.019987:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776457.019988:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37400. 02000000:00000001:1.0:1775776457.019991:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776457.019992:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776457.019993:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72a00. 00000100:00000001:1.0:1775776457.019995:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776457.019996:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776457.019998:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776457.020003:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776457.020004:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776457.020005:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776457.020007:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776457.020008:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776457.020010:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70700 x1862022710415872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.020015:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776457.020016:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776457.020017:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776457.020018:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776457.020020:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776457.020022:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710415872:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776457.020024:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776457.020025:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776457.020026:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776457.020027:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776457.020029:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4dc00. 02000000:00000001:1.0:1775776457.020030:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776457.020033:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776457.020034:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776457.020037:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776457.020039:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710415872, portal 10 00000100:00000001:1.0:1775776457.020040:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776457.020041:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849005824 : -123823860545792 : ffff8f6202c70700) 00000100:00000040:1.0:1775776457.020045:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70700 x1862022710415872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.020059:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776457.020063:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776457.020065:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776457.020067:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710415872, offset 0 00000400:00000200:1.0:1775776457.020070:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776457.020072:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776457.020075:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776457.020077:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776457.020079:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776457.020081:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776457.020083:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776457.020085:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453801:453801:256:4294967295] 192.168.201.115@tcp LPNI seq info [453726:453726:8:4294967295] 00000400:00000200:1.0:1775776457.020087:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776457.020092:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776457.020094:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776457.020096:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776457.020099:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776457.020101:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775776457.020104:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776457.020106:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775776457.020109:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776457.020110:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776457.020112:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776457.020114:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776457.020117:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776457.020118:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776457.020119:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776457.020122:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70700 x1862022710415872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.020127:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776457.020129:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70700 x1862022710415872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776482 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.020155:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776457.020156:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776457.020158:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776457.020161:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776457.020163:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776457.020165:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308012044 00000100:00000001:1.0:1775776457.020167:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308015878 00000100:00000001:1.0:1775776457.020168:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308012044 00000100:00000001:1.0:1775776457.020170:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308015878 00000100:00000001:1.0:1775776457.020172:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776457.020174:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776457.020175:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776457.020178:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776457/real 1775776457] req@ffff8f6202c70700 x1862022710415872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776457.020194:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776457.020195:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776457.020197:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776457.020198:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776457.020199:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776457.020201:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776457.020203:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70700 x1862022710415872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776457.020208:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70700 x1862022710415872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.020212:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776457.020213:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776457.020214:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776457.020215:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776457.020217:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70700 x1862022710415872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.020222:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776457.020224:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70700 x1862022710415872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.020229:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776457.020230:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776457.020231:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776457.020232:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776457.020234:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776457.020237:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776457.020239:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776457.020241:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776457.020243:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776457.020245:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70700 x1862022710415872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776457.020250:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710415872:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776457.020252:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776457.020254:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70a80 x1862022710416000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.020258:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776457.020259:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776457.020261:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776457.020262:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776457.020264:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776457.020265:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710416000:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776457.020267:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776457.020268:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776457.020269:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776457.020270:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776457.020273:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776457.020274:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776457.020276:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776457.020278:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776457.020279:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776457.020281:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710416000, portal 10 00000100:00000001:1.0:1775776457.020283:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776457.020283:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849006720 : -123823860544896 : ffff8f6202c70a80) 00000100:00000040:1.0:1775776457.020287:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70a80 x1862022710416000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.020291:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776457.020293:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776457.020294:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776457.020296:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710416000, offset 0 00000400:00000200:1.0:1775776457.020298:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776457.020300:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776457.020303:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776457.020304:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776457.020306:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776457.020308:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776457.020309:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776457.020312:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453802:453802:256:4294967295] 192.168.201.115@tcp LPNI seq info [453727:453727:8:4294967295] 00000400:00000200:1.0:1775776457.020314:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776457.020318:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776457.020321:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776457.020323:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776457.020326:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776457.020328:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775776457.020330:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776457.020332:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775776457.020334:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776457.020336:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776457.020338:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776457.020340:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776457.020342:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776457.020343:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776457.020344:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776457.020346:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70a80 x1862022710416000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776482 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.020354:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776457.020357:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70a80 x1862022710416000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776482 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.020370:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776457.020371:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776457.020374:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776457.020377:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776457.020381:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776457.020383:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308012044 00000100:00000001:1.0:1775776457.020385:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308015878 00000100:00000001:1.0:1775776457.020387:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308012044 00000100:00000001:1.0:1775776457.020390:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308015878 00000100:00000001:1.0:1775776457.020392:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776457.020394:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776457.020396:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776457.020400:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776457/real 1775776457] req@ffff8f6202c70a80 x1862022710416000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776457.020409:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776457.020410:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776457.020412:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776457.020415:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776457.020417:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776457.020419:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776457.020422:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70a80 x1862022710416000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776457.020431:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70a80 x1862022710416000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.020435:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776457.020437:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776457.020438:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776457.020439:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776457.020441:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70a80 x1862022710416000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.020446:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776457.020448:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70a80 x1862022710416000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.020453:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776457.020454:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776457.020455:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776457.020456:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776457.020457:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776457.020458:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776457.020460:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776457.020462:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776457.020463:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776457.020465:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70a80 x1862022710416000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776457.020470:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710416000:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776457.020472:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776457.020473:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776457.020475:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70700 x1862022710415872/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.020479:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776457.020480:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776457.020482:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4dc00. 02000000:00000001:1.0:1775776457.020483:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776457.020484:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776457.020485:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776457.020487:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776457.020488:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35800. 02000000:00000001:1.0:1775776457.020490:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776457.020491:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776457.020492:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70700. 00000100:00000001:1.0:1775776457.020494:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776457.020495:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776457.020496:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776457.020497:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70a80 x1862022710416000/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776482 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776457.020502:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776457.020503:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776457.020503:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776457.020505:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776457.020505:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776457.020506:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776457.020507:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776457.020508:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35400. 02000000:00000001:1.0:1775776457.020509:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776457.020510:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776457.020510:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70a80. 00000100:00000001:1.0:1775776457.020512:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776457.020512:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776457.020513:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776457.020516:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776457.020517:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776457.020518:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776457.020519:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776457.147264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776457.147269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776457.147273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776457.147274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776457.147277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776457.147278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776457.147347:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776457.147351:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776457.147355:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776458.043263:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776458.043271:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776458.043273:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776458.043277:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776458.043280:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776458.043283:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776458.043286:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776458.043287:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776458.043289:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776458.171275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776458.171280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776458.171284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776458.171285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776458.171288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776458.171289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776458.171362:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776458.171367:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776458.171371:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776459.067263:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776459.067268:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776459.067270:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776459.067274:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776459.067277:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776459.067280:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776459.067282:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776459.067284:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776459.067286:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776459.195263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776459.195270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776459.195274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776459.195275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776459.195278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776459.195279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776459.195331:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776459.195336:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776459.195340:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776460.091275:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776460.091280:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776460.091282:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776460.091286:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776460.091289:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776460.091291:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776460.091293:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776460.091295:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776460.091297:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776460.219278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776460.219283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776460.219287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776460.219288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776460.219291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776460.219292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776460.219310:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776460.219313:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776460.219316:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776460.923255:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308015878, at 4308015936 00000100:00000040:0.0:1775776460.923265:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308015878, at 4308015936 00000100:00000010:1.0:1775776460.923272:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2a20. 00000100:00000010:0.0:1775776460.923273:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e28a0. 00000100:00000001:0.0:1775776461.115246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776461.115254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776461.115256:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776461.115262:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776461.115266:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776461.115270:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776461.115283:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776461.115286:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776461.115292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776461.243212:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776461.243216:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776461.243219:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776461.243220:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776461.243223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776461.243224:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776461.243295:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776461.243299:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776461.243304:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776462.139284:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776462.139293:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1775776462.139294:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776462.139299:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775776462.139300:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776462.139304:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776462.139306:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775776462.139309:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776462.139312:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776462.139314:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776462.139318:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775776462.139322:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13344 00000100:00000001:2.0:1775776462.139323:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776462.139328:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776462.139331:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776462.139336:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776462.139343:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776462.139347:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776462.139351:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776462.139355:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776462.139358:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776462.139361:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776462.139365:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776462.139368:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776462.139371:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776462.139374:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776462.139376:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776462.139379:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776462.139382:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776462.139384:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776462.139386:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776462.139388:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776462.139392:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776462.139395:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776462.139398:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776462.139402:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776462.139405:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776462.139408:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776462.139410:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776462.139413:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776462.139423:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061c00. 00000020:00000040:0.0:1775776462.139427:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776462.139431:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776462.139433:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776462.139436:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776462.139439:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776462.139443:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776462.139452:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4400. 00000100:00000001:0.0:1775776462.139459:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776462.139468:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061c00 x1862022710416128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776462.139479:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776462.139484:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776462.139489:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076a44879] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061c00 x1862022710416128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776462.139512:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776462.139515:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776462.139519:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776462.139520:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776462.139522:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776462.139525:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776462.139527:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13344 00000400:00000200:0.0:1775776462.139532:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776462.139535:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776462.139537:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776462.139541:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776462.139543:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776462.139546:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776462.139549:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776462.139552:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776462.139554:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776462.139555:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776462.139558:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:1.0:1775776462.139558:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776462.139563:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1775776462.139564:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776462.139566:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1775776462.139569:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:1.0:1775776462.139574:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061c00 x1862022710416128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776462.139577:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776462.139584:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776462.139589:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776462.139589:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776462.139592:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776462.139594:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776462.139596:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:0.0:1775776462.139598:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:1.0:1775776462.139599:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776462.139604:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776462.139606:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00100000:1.0:1775776462.139609:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710416128:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776462.139613:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776462.139614:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776462.139618:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776462.139619:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776462.139622:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776462.139623:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1775776462.139625:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:1.0:1775776462.139627:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4dc00. 02000000:00000001:0.0:1775776462.139631:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776462.139631:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776462.139637:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775776462.139641:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000008:00100000:0.0:1775776462.139643:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000010:1.0:1775776462.139645:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000008:00000001:0.0:1775776462.139650:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776462.139653:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000001:0.0:1775776462.139654:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776462.139662:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061880. 00000100:00000200:1.0:1775776462.139662:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710416128, portal 25 00000100:00000001:1.0:1775776462.139666:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000040:0.0:1775776462.139668:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000001:1.0:1775776462.139669:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574560256 : -123823134991360 : ffff8f622e061c00) 02000000:00000001:0.0:1775776462.139673:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776462.139676:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776462.139678:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061c00 x1862022710416128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776462.139680:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776462.139683:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776462.139686:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776462.139692:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776462.139696:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776462.139699:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 02000000:00000010:0.0:1775776462.139703:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000200:1.0:1775776462.139704:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710416128, offset 0 00000100:00000001:0.0:1775776462.139710:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776462.139716:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00100000:0.0:1775776462.139719:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061880 x1862022710416256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776462.139729:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776462.139734:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776462.139738:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776462.139740:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776462.139745:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:0.0:1775776462.139746:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005afd1f27] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061880 x1862022710416256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776462.139750:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776462.139757:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776462.139763:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:0.0:1775776462.139765:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776462.139770:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776462.139771:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453803:453803:256:4294967295] 192.168.201.115@tcp LPNI seq info [453728:453728:8:4294967295] 00000100:00080000:0.0:1775776462.139779:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776462.139783:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1775776462.139783:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00080000:0.0:1775776462.139787:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776462.139792:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776462.139797:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13344 00000400:00000200:1.0:1775776462.139803:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000400:00000200:0.0:1775776462.139815:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000200:1.0:1775776462.139820:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776462.139824:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000800:00000010:1.0:1775776462.139825:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000100:00080000:0.0:1775776462.139829:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776462.139835:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000800:00000200:1.0:1775776462.139837:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775776462.139841:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.0:1775776462.139842:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000100:00000040:0.0:1775776462.139847:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776462.139853:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000200:1.1:1775776462.139853:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:0.0:1775776462.139858:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000800:00000200:1.1:1775776462.139860:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000020:00000040:0.0:1775776462.139861:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776462.139864:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776462.139868:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000800:00000100:1.0:1775776462.139869:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776462.139874:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776462.139881:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:0.0:1775776462.139885:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776462.139889:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000020:00000001:0.0:1775776462.139890:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775776462.139897:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776462.139897:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000100:00000001:0.0:1775776462.139903:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1775776462.139903:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000040:0.0:1775776462.139909:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776462.139911:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1775776462.139913:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000200:1.0:1775776462.139917:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061c00 x1862022710416128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775776462.139918:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775776462.139927:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776462.139932:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776462.139933:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1775776462.139937:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1775776462.139938:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061c00 x1862022710416128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776487 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775776462.139943:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776462.139946:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776462.139948:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776462.139949:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776462.139953:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776462.139955:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1775776462.139956:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776462.139962:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776462.139962:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776462.139964:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308020998 00000100:00000001:0.0:1775776462.139967:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776462.139970:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2a20. 00000100:00000010:0.0:1775776462.139973:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060700. 00000020:00000040:0.0:1775776462.139977:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000100:00000001:1.0:1775776462.139977:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1775776462.139983:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776462.139985:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776462.139985:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308020998 00000100:00000001:0.0:1775776462.139988:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000010:1.0:1775776462.139991:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e28a0. 02000000:00000001:0.0:1775776462.139992:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776462.139996:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776462.139997:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776462.140003:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:0.0:1775776462.140005:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6400. 00000100:00000001:1.0:1775776462.140005:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776462.140007:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1775776462.140012:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776462.140012:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776462/real 1775776462] req@ffff8f622e061c00 x1862022710416128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776462.140023:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:0.0:1775776462.140025:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060700 x1862022710416384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1775776462.140025:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776462.140027:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776462.140031:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776462.140034:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776462.140038:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776462.140044:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061c00 x1862022710416128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776462.140046:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776462.140051:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776462.140055:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061c00 x1862022710416128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776462.140061:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000225c8695] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060700 x1862022710416384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776462.140063:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776462.140065:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776462.140070:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776462.140072:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776462.140079:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061c00 x1862022710416128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776462.140084:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776462.140106:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776462.140111:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776462.140112:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776462.140114:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1775776462.140119:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061c00 x1862022710416128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776462.140120:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776462.140129:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13354) 00000100:00000001:1.0:1775776462.140129:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776462.140160:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776462.140163:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776462.140164:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776462.140167:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776462.140169:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776462.140173:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776462.140179:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776462.140183:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776462.140188:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061c00 x1862022710416128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776462.140199:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710416128:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776462.140204:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776462.140206:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776462.140209:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061c00 x1862022710416128/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776462.140217:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776462.140219:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776462.140221:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4dc00. 02000000:00000001:1.0:1775776462.140225:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776462.140227:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776462.140228:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776462.140230:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776462.140233:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:1.0:1775776462.140236:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776462.140238:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776462.140240:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061c00. 00000100:00000001:1.0:1775776462.140243:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776462.140244:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776462.140247:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776462.140253:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776462.140255:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776462.140257:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776462.140258:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776462.140259:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776462.140262:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061880 x1862022710416256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776462.140267:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776462.140268:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776462.140270:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776462.140271:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776462.140273:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776462.140274:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710416256:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776462.140277:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776462.140278:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776462.140279:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776462.140281:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776462.140283:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4dc00. 02000000:00000001:1.0:1775776462.140284:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776462.140287:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776462.140290:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776462.140292:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776462.140295:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710416256, portal 10 00000100:00000001:1.0:1775776462.140297:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776462.140298:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574559360 : -123823134992256 : ffff8f622e061880) 00000100:00000040:1.0:1775776462.140305:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061880 x1862022710416256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776462.140312:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776462.140314:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776462.140317:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776462.140319:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710416256, offset 0 00000400:00000200:1.0:1775776462.140324:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776462.140328:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776462.140332:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776462.140335:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776462.140338:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776462.140341:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776462.140343:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776462.140347:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453804:453804:256:4294967295] 192.168.201.115@tcp LPNI seq info [453729:453729:8:4294967295] 00000400:00000200:1.0:1775776462.140351:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776462.140358:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776462.140364:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776462.140368:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776462.140372:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776462.140375:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775776462.140380:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776462.140383:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775776462.140387:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776462.140389:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776462.140392:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776462.140395:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776462.140398:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776462.140400:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776462.140403:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776462.140407:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061880 x1862022710416256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776462.140415:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776462.140418:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061880 x1862022710416256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776487 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776462.140425:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776462.140427:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776462.140430:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776462.140434:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776462.140438:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776462.140440:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308017164 00000100:00000001:1.0:1775776462.140442:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308020998 00000100:00000001:1.0:1775776462.140445:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308017164 00000100:00000001:1.0:1775776462.140447:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308020998 00000100:00000001:1.0:1775776462.140449:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776462.140452:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776462.140453:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776462.140458:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776462/real 1775776462] req@ffff8f622e061880 x1862022710416256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776462.140466:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776462.140468:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776462.140469:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776462.140472:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776462.140474:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776462.140477:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776462.140479:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061880 x1862022710416256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776462.140488:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061880 x1862022710416256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776462.140495:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776462.140496:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776462.140498:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776462.140499:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776462.140503:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061880 x1862022710416256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776462.140510:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776462.140514:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061880 x1862022710416256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776462.140522:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776462.140524:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776462.140526:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776462.140527:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776462.140529:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776462.140534:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776462.140537:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776462.140540:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776462.140542:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776462.140547:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061880 x1862022710416256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776462.140556:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710416256:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776462.140560:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776462.140563:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e060700 x1862022710416384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776462.140570:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776462.140572:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776462.140574:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776462.140576:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776462.140578:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776462.140581:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e060700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710416384:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776462.140584:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776462.140586:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776462.140588:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776462.140590:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776462.140594:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776462.140596:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776462.140600:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776462.140602:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776462.140605:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776462.140608:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710416384, portal 10 00000100:00000001:1.0:1775776462.140610:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776462.140612:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574554880 : -123823134996736 : ffff8f622e060700) 00000100:00000040:1.0:1775776462.140618:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e060700 x1862022710416384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776462.140625:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776462.140628:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776462.140630:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776462.140633:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710416384, offset 0 00000400:00000200:1.0:1775776462.140637:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776462.140642:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776462.140645:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776462.140648:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776462.140651:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776462.140653:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776462.140656:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776462.140659:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453805:453805:256:4294967295] 192.168.201.115@tcp LPNI seq info [453730:453730:8:4294967295] 00000400:00000200:1.0:1775776462.140663:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776462.140670:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776462.140675:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776462.140678:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776462.140682:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776462.140685:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775776462.140690:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776462.140692:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775776462.140696:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776462.140698:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776462.140701:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776462.140705:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776462.140708:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776462.140710:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776462.140713:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776462.140716:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e060700 x1862022710416384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776487 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776462.140725:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776462.140728:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e060700 x1862022710416384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776487 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776462.140732:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776462.140733:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776462.140735:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776462.140738:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776462.140740:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776462.140742:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308017164 00000100:00000001:1.0:1775776462.140744:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308020998 00000100:00000001:1.0:1775776462.140745:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308017164 00000100:00000001:1.0:1775776462.140747:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308020998 00000100:00000001:1.0:1775776462.140748:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776462.140749:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776462.140750:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776462.140752:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776462/real 1775776462] req@ffff8f622e060700 x1862022710416384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776462.140760:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776462.140762:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776462.140763:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776462.140766:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776462.140768:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776462.140770:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776462.140773:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e060700 x1862022710416384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776462.140782:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e060700 x1862022710416384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776462.140789:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776462.140791:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776462.140793:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776462.140794:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776462.140798:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e060700 x1862022710416384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776462.140806:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776462.140810:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e060700 x1862022710416384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776462.140815:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776462.140817:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776462.140818:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776462.140819:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776462.140821:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776462.140822:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776462.140824:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776462.140826:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776462.140827:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776462.140829:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e060700 x1862022710416384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776462.140835:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e060700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710416384:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776462.140837:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776462.140838:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776462.140840:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061880 x1862022710416256/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776462.140844:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776462.140846:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776462.140848:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4dc00. 02000000:00000001:1.0:1775776462.140850:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776462.140851:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776462.140853:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776462.140855:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776462.140857:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:1.0:1775776462.140859:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776462.140860:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776462.140862:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061880. 00000100:00000001:1.0:1775776462.140864:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776462.140865:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776462.140867:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776462.140870:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e060700 x1862022710416384/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776487 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776462.140884:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776462.140885:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776462.140887:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:1.0:1775776462.140890:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776462.140891:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776462.140892:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776462.140894:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776462.140896:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:1.0:1775776462.140898:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776462.140899:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776462.140901:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e060700. 00000100:00000001:1.0:1775776462.140902:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776462.140903:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776462.140904:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776462.140907:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776462.140908:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776462.140910:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776462.140911:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776462.267248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776462.267252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776462.267256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776462.267258:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776462.267260:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776462.267263:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776462.267265:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776462.267267:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776462.267268:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776462.267270:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776462.267280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776462.267281:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776462.267282:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776462.267283:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776462.267284:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776462.267285:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776462.267289:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776462.267291:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776462.267349:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776462.267352:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776462.267355:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776463.163249:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776463.163256:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776463.163258:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776463.163263:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776463.163267:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776463.163271:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776463.163274:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776463.163276:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776463.163279:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776463.291213:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776463.291218:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776463.291222:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776463.291223:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776463.291226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776463.291227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776463.291290:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776463.291295:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776463.291299:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776464.187245:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776464.187251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776464.187252:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776464.187257:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776464.187259:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776464.187262:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776464.187265:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776464.187267:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776464.187268:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776464.315248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776464.315253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776464.315256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776464.315257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776464.315260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776464.315261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776464.315314:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776464.315318:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776464.315320:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776465.211251:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776465.211258:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776465.211261:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776465.211267:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776465.211271:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776465.211276:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776465.211278:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776465.211281:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776465.211284:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776465.339294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776465.339305:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776465.339308:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776465.339312:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776465.339318:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776465.339321:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776465.339323:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776465.339326:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776465.339330:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776465.339332:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776465.339336:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776465.339337:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776465.339391:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776465.339394:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776465.339396:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776466.043256:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308020998, at 4308021056 00000100:00000010:1.0:1775776466.043274:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e28a0. 00000100:00000040:0.0:1775776466.043278:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308020998, at 4308021056 00000100:00000010:0.0:1775776466.043293:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2a20. 00000100:00000001:2.0:1775776466.235277:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776466.235284:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776466.235286:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776466.235292:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776466.235297:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776466.235300:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776466.235303:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776466.235306:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776466.235309:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776466.363267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776466.363275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776466.363283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776466.363284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776466.363288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776466.363290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776466.363361:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776466.363366:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776466.363369:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776467.259227:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776467.259233:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776467.259236:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776467.259242:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776467.259245:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1775776467.259248:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776467.259249:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776467.259253:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:1.0:1775776467.259255:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776467.259255:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776467.259259:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1775776467.259259:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776467.259263:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776467.259266:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776467.259273:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13349 00000400:00000200:1.0:1775776467.259283:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776467.259289:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776467.259292:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776467.259294:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776467.259297:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776467.259299:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776467.259300:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:1.0:1775776467.259303:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776467.259305:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:1.0:1775776467.259307:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776467.259309:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:1.0:1775776467.259310:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:1.0:1775776467.259312:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:1.0:1775776467.259314:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776467.259315:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776467.259317:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776467.259318:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:1.0:1775776467.259320:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:1.0:1775776467.259322:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776467.259323:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776467.259326:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776467.259329:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776467.259331:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776467.259332:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776467.259333:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776467.259343:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6234918000. 00000020:00000040:1.0:1775776467.259346:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:1.0:1775776467.259350:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776467.259351:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776467.259354:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776467.259357:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776467.259362:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776467.259371:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4e800. 00000100:00000001:1.0:1775776467.259379:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776467.259389:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6234918000 x1862022710416512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776467.259400:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776467.259405:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776467.259408:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000061c0fbd7] to pc [ptlrpcd_rcv+-1] req@ffff8f6234918000 x1862022710416512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776467.259431:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776467.259434:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776467.259445:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776467.259447:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776467.259449:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776467.259452:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776467.259456:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13349 00000100:00000001:0.0:1775776467.259459:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1775776467.259460:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776467.259465:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1775776467.259466:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:0.0:1775776467.259469:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1775776467.259469:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776467.259474:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776467.259476:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776467.259479:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000040:0.0:1775776467.259480:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918000 x1862022710416512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776467.259482:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:1.0:1775776467.259488:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776467.259490:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:1.0:1775776467.259494:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775776467.259496:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:1.0:1775776467.259497:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:0.0:1775776467.259499:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776467.259501:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:0.0:1775776467.259504:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1775776467.259504:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:0.0:1775776467.259508:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1775776467.259510:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 02000000:00000001:0.0:1775776467.259513:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776467.259514:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:0.0:1775776467.259520:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710416512:192.168.201.115@tcp:250:kworker.0 00000100:00000040:1.0:1775776467.259520:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776467.259525:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:0.0:1775776467.259526:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:1.0:1775776467.259529:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:0.0:1775776467.259532:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1775776467.259532:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000001:0.0:1775776467.259535:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776467.259536:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776467.259538:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1775776467.259539:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:0.0:1775776467.259545:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7800. 02000000:00000001:1.0:1775776467.259545:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776467.259549:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776467.259549:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776467.259554:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:0.0:1775776467.259557:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000008:00100000:1.0:1775776467.259558:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000010:0.0:1775776467.259561:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000008:00000001:1.0:1775776467.259561:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776467.259565:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776467.259570:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000010:1.0:1775776467.259571:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623491b800. 00000020:00000040:1.0:1775776467.259574:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776467.259576:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000200:0.0:1775776467.259579:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710416512, portal 25 00000100:00000001:0.0:1775776467.259581:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776467.259583:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684358656 : -123823025192960 : ffff8f6234918000) 00000100:00000040:0.0:1775776467.259587:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918000 x1862022710416512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776492 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1775776467.259588:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776467.259593:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776467.259596:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1775776467.259597:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1775776467.259599:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776467.259602:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776467.259605:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 02000000:00000010:1.0:1775776467.259607:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4dc00. 00000100:00000200:0.0:1775776467.259608:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710416512, offset 0 00000100:00000001:1.0:1775776467.259612:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776467.259615:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00100000:1.0:1775776467.259618:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623491b800 x1862022710416640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776467.259624:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1775776467.259625:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776467.259627:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776467.259631:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000958ac43a] to pc [ptlrpcd_rcv+-1] req@ffff8f623491b800 x1862022710416640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776467.259634:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776467.259642:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1775776467.259642:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776467.259646:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775776467.259649:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00080000:1.0:1775776467.259652:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776467.259654:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1775776467.259655:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00080000:1.0:1775776467.259657:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1775776467.259659:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:1.0:1775776467.259661:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776467.259664:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13349 00000400:00000200:0.0:1775776467.259668:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453806:453806:256:4294967295] 192.168.201.115@tcp LPNI seq info [453731:453731:8:4294967295] 00000400:00000200:0.0:1775776467.259672:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776467.259679:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776467.259686:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776467.259686:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776467.259691:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000800:00000010:0.0:1775776467.259692:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000100:00080000:1.0:1775776467.259693:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776467.259696:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776467.259698:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776467.259701:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:0.0:1775776467.259702:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:1.0:1775776467.259704:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000200:0.0:1775776467.259707:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000020:00000001:1.0:1775776467.259707:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776467.259708:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:1.0:1775776467.259711:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776467.259714:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:1.0:1775776467.259717:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:0.1:1775776467.259717:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:1.0:1775776467.259721:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:0.1:1775776467.259722:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000100:00000040:1.0:1775776467.259725:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776467.259727:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:0.0:1775776467.259730:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000040:1.0:1775776467.259731:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000010:0.0:1775776467.259734:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000100:00000001:1.0:1775776467.259734:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:1.0:1775776467.259735:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:1.0:1775776467.259738:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776467.259739:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776467.259740:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1775776467.259741:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:1.0:1775776467.259742:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776467.259745:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776467.259748:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1775776467.259748:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776467.259750:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1775776467.259753:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000008:00000001:1.0:1775776467.259753:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776467.259754:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776467.259757:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776467.259762:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:1.0:1775776467.259764:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6234918a80. 00000100:00000200:0.0:1775776467.259768:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918000 x1862022710416512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776492 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1775776467.259768:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776467.259773:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776467.259775:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776467.259779:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776467.259781:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776467.259784:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776467.259785:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000010:1.0:1775776467.259789:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4d000. 00000100:00000040:0.0:1775776467.259790:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918000 x1862022710416512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776492 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776467.259794:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776467.259803:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6234918a80 x1862022710416768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776467.259804:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776467.259808:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776467.259814:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776467.259816:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776467.259817:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776467.259822:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:1.0:1775776467.259823:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000026c0c0d3] to pc [ptlrpcd_rcv+-1] req@ffff8f6234918a80 x1862022710416768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.259826:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308026118 00000100:00000001:1.0:1775776467.259834:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776467.259835:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3480. 00000100:00000040:1.0:1775776467.259836:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1775776467.259840:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:1.0:1775776467.259843:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1775776467.259845:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1775776467.259846:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000040:1.0:1775776467.259849:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13359) 00000100:00000001:0.0:1775776467.259856:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308026118 00000100:00000010:0.0:1775776467.259861:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c34e0. 00000100:00000001:0.0:1775776467.259864:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776467.259869:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776467.259872:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776467.259875:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776467.259880:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776467/real 1775776467] req@ffff8f6234918000 x1862022710416512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776467.259891:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776467.259892:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776467.259894:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776467.259896:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776467.259897:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776467.259899:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776467.259902:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918000 x1862022710416512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776467.259907:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918000 x1862022710416512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.259911:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776467.259913:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776467.259914:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776467.259916:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776467.259918:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918000 x1862022710416512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.259921:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776467.259924:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918000 x1862022710416512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.259929:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776467.259931:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776467.259932:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776467.259933:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776467.259934:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776467.259936:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776467.259937:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776467.259940:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776467.259942:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776467.259945:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918000 x1862022710416512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776467.259950:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710416512:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776467.259952:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776467.259954:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776467.259955:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918000 x1862022710416512/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.259959:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776467.259961:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776467.259963:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776467.259965:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776467.259966:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776467.259967:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776467.259969:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776467.259971:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4e800. 02000000:00000001:0.0:1775776467.259974:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776467.259975:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776467.259977:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918000. 00000100:00000001:0.0:1775776467.259978:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776467.259979:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776467.259981:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776467.259987:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776467.259988:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776467.259990:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776467.259991:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776467.259992:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776467.259994:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491b800 x1862022710416640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.259998:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776467.259999:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776467.260001:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776467.260015:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776467.260018:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776467.260021:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710416640:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776467.260023:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776467.260024:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776467.260025:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776467.260027:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776467.260029:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776467.260030:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776467.260033:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776467.260034:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776467.260037:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776467.260039:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710416640, portal 10 00000100:00000001:0.0:1775776467.260040:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776467.260041:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684372992 : -123823025178624 : ffff8f623491b800) 00000100:00000040:0.0:1775776467.260046:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491b800 x1862022710416640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776492 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.260050:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776467.260051:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776467.260052:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776467.260054:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710416640, offset 0 00000400:00000200:0.0:1775776467.260056:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776467.260058:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776467.260060:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776467.260062:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776467.260064:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776467.260065:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776467.260067:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776467.260069:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453807:453807:256:4294967295] 192.168.201.115@tcp LPNI seq info [453732:453732:8:4294967295] 00000400:00000200:0.0:1775776467.260072:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776467.260076:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776467.260078:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776467.260081:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776467.260083:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776467.260085:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775776467.260088:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776467.260089:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1775776467.260091:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776467.260093:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776467.260094:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776467.260097:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776467.260098:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776467.260100:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776467.260101:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776467.260103:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491b800 x1862022710416640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776492 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.260108:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776467.260109:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491b800 x1862022710416640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776492 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.260113:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776467.260114:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776467.260116:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776467.260119:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776467.260120:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776467.260121:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308022283 00000100:00000001:0.0:1775776467.260124:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308026118 00000100:00000001:0.0:1775776467.260125:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308022283 00000100:00000001:0.0:1775776467.260126:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308026118 00000100:00000001:0.0:1775776467.260128:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776467.260129:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776467.260130:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776467.260132:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776467/real 1775776467] req@ffff8f623491b800 x1862022710416640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776467.260157:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776467.260158:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776467.260159:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776467.260161:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776467.260162:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776467.260163:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776467.260166:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491b800 x1862022710416640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776467.260171:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491b800 x1862022710416640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.260174:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776467.260175:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776467.260176:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776467.260176:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776467.260178:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491b800 x1862022710416640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.260182:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776467.260185:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491b800 x1862022710416640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.260189:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776467.260190:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776467.260191:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776467.260191:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776467.260192:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776467.260196:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776467.260198:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776467.260199:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776467.260200:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776467.260203:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491b800 x1862022710416640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776467.260208:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710416640:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776467.260210:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776467.260212:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918a80 x1862022710416768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.260215:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776467.260216:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776467.260217:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776467.260218:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776467.260219:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776467.260221:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710416768:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776467.260222:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776467.260223:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776467.260224:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776467.260225:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776467.260227:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776467.260228:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776467.260230:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776467.260231:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776467.260232:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776467.260234:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710416768, portal 10 00000100:00000001:0.0:1775776467.260236:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776467.260236:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684361344 : -123823025190272 : ffff8f6234918a80) 00000100:00000040:0.0:1775776467.260240:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918a80 x1862022710416768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776492 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.260244:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776467.260245:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776467.260246:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776467.260248:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710416768, offset 0 00000400:00000200:0.0:1775776467.260251:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776467.260253:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776467.260255:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776467.260256:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776467.260258:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776467.260259:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776467.260261:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776467.260263:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453808:453808:256:4294967295] 192.168.201.115@tcp LPNI seq info [453733:453733:8:4294967295] 00000400:00000200:0.0:1775776467.260265:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776467.260269:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776467.260272:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776467.260273:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776467.260276:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776467.260277:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775776467.260280:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776467.260281:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1775776467.260284:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776467.260285:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776467.260287:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776467.260289:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776467.260298:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776467.260299:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776467.260301:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776467.260303:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918a80 x1862022710416768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776492 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.260307:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776467.260309:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918a80 x1862022710416768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776492 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.260313:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776467.260314:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776467.260316:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776467.260317:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776467.260319:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776467.260321:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308022284 00000100:00000001:0.0:1775776467.260323:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308026118 00000100:00000001:0.0:1775776467.260324:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308022284 00000100:00000001:0.0:1775776467.260325:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308026118 00000100:00000001:0.0:1775776467.260326:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776467.260327:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776467.260328:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776467.260330:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776467/real 1775776467] req@ffff8f6234918a80 x1862022710416768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776467.260334:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776467.260335:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776467.260336:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776467.260338:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776467.260339:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776467.260340:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776467.260341:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918a80 x1862022710416768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776467.260346:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918a80 x1862022710416768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.260350:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776467.260351:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776467.260352:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776467.260352:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776467.260354:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918a80 x1862022710416768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.260357:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776467.260359:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918a80 x1862022710416768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.260363:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776467.260365:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776467.260366:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776467.260367:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776467.260368:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776467.260370:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776467.260373:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776467.260375:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776467.260378:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776467.260381:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918a80 x1862022710416768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776467.260388:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710416768:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776467.260392:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776467.260394:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776467.260396:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491b800 x1862022710416640/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.260403:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776467.260405:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776467.260406:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776467.260408:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776467.260410:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776467.260411:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776467.260413:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776467.260415:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4dc00. 02000000:00000001:0.0:1775776467.260417:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776467.260419:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776467.260420:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491b800. 00000100:00000001:0.0:1775776467.260423:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776467.260423:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776467.260425:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776467.260427:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918a80 x1862022710416768/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776492 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776467.260435:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776467.260436:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776467.260437:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776467.260438:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776467.260439:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776467.260439:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776467.260440:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776467.260441:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4d000. 02000000:00000001:0.0:1775776467.260442:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776467.260443:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776467.260443:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918a80. 00000100:00000001:0.0:1775776467.260444:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776467.260445:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776467.260446:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776467.260448:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776467.260449:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776467.260450:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776467.260451:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776467.387248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776467.387253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776467.387257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776467.387258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776467.387261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776467.387262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776467.387358:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776467.387365:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776467.387370:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776468.283262:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776468.283268:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776468.283270:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776468.283274:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776468.283277:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776468.283280:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776468.283282:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776468.283283:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776468.283285:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776468.411237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776468.411243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776468.411251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776468.411253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776468.411257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776468.411258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776468.411320:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776468.411324:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776468.411327:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776469.307193:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776469.307201:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776469.307203:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776469.307208:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776469.307211:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776469.307214:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776469.307216:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776469.307218:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776469.307220:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776469.435245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776469.435250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776469.435256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776469.435257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776469.435262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776469.435264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776469.435338:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776469.435342:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776469.435347:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776470.331272:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776470.331279:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776470.331280:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776470.331285:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776470.331289:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776470.331292:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776470.331295:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776470.331297:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776470.331300:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776470.459246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776470.459252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776470.459256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776470.459256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776470.459259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776470.459260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776470.459320:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776470.459324:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776470.459326:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776471.163229:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308026118, at 4308026176 00000100:00000040:0.0:1775776471.163238:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308026118, at 4308026176 00000100:00000010:1.0:1775776471.163245:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c34e0. 00000100:00000010:0.0:1775776471.163246:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776471.355251:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776471.355258:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776471.355259:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776471.355263:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776471.355266:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776471.355269:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776471.355271:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776471.355273:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776471.355274:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776471.483245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776471.483249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776471.483253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776471.483256:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776471.483257:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776471.483260:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776471.483262:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776471.483263:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776471.483265:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776471.483266:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776471.483269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776471.483270:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776471.483270:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776471.483271:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776471.483279:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776471.483279:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776471.483280:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776471.483282:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776471.483358:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776471.483363:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776471.483366:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776472.379219:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775776472.379221:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776472.379226:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1775776472.379227:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776472.379230:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776472.379230:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776472.379234:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775776472.379236:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776472.379238:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775776472.379240:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776472.379244:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775776472.379244:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13354 00000100:00000001:0.0:1775776472.379249:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776472.379252:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1775776472.379256:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776472.379258:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776472.379262:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776472.379265:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776472.379268:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776472.379270:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776472.379273:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776472.379274:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776472.379277:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776472.379278:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776472.379281:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776472.379282:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776472.379284:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776472.379285:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776472.379287:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776472.379288:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776472.379290:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776472.379291:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776472.379292:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776472.379294:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776472.379296:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776472.379297:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776472.379299:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776472.379301:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776472.379302:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776472.379304:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776472.379310:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775776472.379312:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776472.379315:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776472.379317:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776472.379318:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776472.379320:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776472.379323:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776472.379330:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000001:3.0:1775776472.379335:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776472.379341:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710416896/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776472.379348:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776472.379351:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776472.379354:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710416896/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776472.379368:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776472.379370:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776472.379372:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776472.379373:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776472.379374:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776472.379376:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776472.379378:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13354 00000400:00000200:3.0:1775776472.379380:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776472.379381:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776472.379383:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776472.379385:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776472.379386:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776472.379388:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776472.379390:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776472.379392:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776472.379393:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776472.379394:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776472.379395:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:0.0:1775776472.379397:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1775776472.379397:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776472.379400:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775776472.379401:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775776472.379404:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776472.379405:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776472.379406:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776472.379408:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776472.379409:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776472.379412:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000040:0.0:1775776472.379413:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71500 x1862022710416896/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775776472.379421:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776472.379425:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776472.379427:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1775776472.379428:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775776472.379431:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776472.379433:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776472.379435:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776472.379435:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776472.379438:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776472.379438:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775776472.379441:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775776472.379441:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776472.379445:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776472.379449:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710416896:192.168.201.115@tcp:250:kworker.0 00000100:00000001:3.0:1775776472.379449:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776472.379451:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775776472.379455:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000001:0.0:1775776472.379456:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776472.379458:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776472.379460:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776472.379460:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776472.379461:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776472.379463:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776472.379464:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776472.379466:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776472.379467:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776472.379469:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:3.0:1775776472.379472:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776472.379473:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 00000100:00100000:3.0:1775776472.379477:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710417024/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1775776472.379478:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776472.379487:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776472.379488:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776472.379491:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776472.379492:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000100:00000040:3.0:1775776472.379494:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710417024/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1775776472.379500:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000001:3.0:1775776472.379501:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776472.379505:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776472.379507:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000200:0.0:1775776472.379508:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710416896, portal 25 00000100:00000001:3.0:1775776472.379508:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776472.379509:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776472.379514:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776472.379514:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775776472.379517:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849009408 : -123823860542208 : ffff8f6202c71500) 00000100:00080000:3.0:1775776472.379518:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13354 00000400:00000200:3.0:1775776472.379522:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:0.0:1775776472.379526:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71500 x1862022710416896/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776497 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775776472.379527:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776472.379531:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776472.379538:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776472.379540:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775776472.379543:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776472.379547:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776472.379550:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000040:3.0:1775776472.379550:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776472.379555:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000200:0.0:1775776472.379556:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710416896, offset 0 00000020:00000001:3.0:1775776472.379562:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776472.379564:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776472.379566:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:0.0:1775776472.379567:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000020:00000040:3.0:1775776472.379571:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776472.379574:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:0.0:1775776472.379578:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000001:3.0:1775776472.379580:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:0.0:1775776472.379586:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000040:3.0:1775776472.379586:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776472.379590:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775776472.379593:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1775776472.379594:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776472.379598:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:0.0:1775776472.379599:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000020:00000040:3.0:1775776472.379603:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:0.0:1775776472.379604:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776472.379609:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00080000:3.0:1775776472.379609:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776472.379613:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776472.379615:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453809:453809:256:4294967295] 192.168.201.115@tcp LPNI seq info [453734:453734:8:4294967295] 02000000:00000001:3.0:1775776472.379616:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776472.379620:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775776472.379621:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776472.379627:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 02000000:00000001:3.0:1775776472.379631:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775776472.379634:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776472.379637:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000010:0.0:1775776472.379640:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000008:00100000:3.0:1775776472.379640:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776472.379644:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776472.379645:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775776472.379648:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000010:3.0:1775776472.379648:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000800:00000200:0.0:1775776472.379652:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000020:00000040:3.0:1775776472.379652:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776472.379655:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776472.379656:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775776472.379658:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775776472.379659:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776472.379662:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:0.1:1775776472.379662:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 02000000:00000001:3.0:1775776472.379663:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1775776472.379666:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776472.379668:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776472.379672:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776472.379675:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776472.379679:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776472.379681:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776472.379685:0:2411:0:(events.c:34:request_out_callback()) Process entered 02000000:00000010:3.0:1775776472.379685:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000100:00000200:0.0:1775776472.379690:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71500 x1862022710416896/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776497 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776472.379690:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776472.379698:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710417152/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776472.379726:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776472.379728:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776472.379729:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776472.379732:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710417152/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776472.379736:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71500 x1862022710416896/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776497 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.379747:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776472.379751:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776472.379752:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776472.379755:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776472.379757:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776472.379758:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776472.379762:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775776472.379765:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775776472.379767:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1775776472.379769:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308031238 00000100:00000040:3.0:1775776472.379772:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13365) 00000100:00000010:0.0:1775776472.379775:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776472.379783:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776472.379789:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308031238 00000100:00000010:0.0:1775776472.379792:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c37e0. 00000100:00000001:0.0:1775776472.379796:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776472.379799:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776472.379801:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776472.379803:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776472.379806:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776472/real 1775776472] req@ffff8f6202c71500 x1862022710416896/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776472.379814:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776472.379815:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776472.379817:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776472.379819:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776472.379820:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776472.379822:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776472.379825:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71500 x1862022710416896/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776472.379832:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71500 x1862022710416896/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.379838:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776472.379840:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776472.379842:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776472.379843:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776472.379846:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71500 x1862022710416896/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.379852:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776472.379855:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71500 x1862022710416896/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.379861:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776472.379864:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776472.379865:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776472.379866:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776472.379868:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776472.379870:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776472.379872:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776472.379876:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776472.379877:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776472.379889:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71500 x1862022710416896/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776472.379898:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710416896:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776472.379901:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776472.379903:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776472.379905:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71500 x1862022710416896/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.379909:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776472.379910:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776472.379924:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776472.379927:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776472.379928:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776472.379929:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776472.379931:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776472.379932:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37800. 02000000:00000001:0.0:1775776472.379935:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776472.379936:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776472.379938:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71500. 00000100:00000001:0.0:1775776472.379940:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776472.379941:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776472.379942:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776472.379947:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776472.379948:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776472.379949:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776472.379950:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776472.379951:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776472.379954:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70380 x1862022710417024/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.379958:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776472.379959:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776472.379961:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776472.379962:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776472.379964:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776472.379965:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710417024:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776472.379967:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776472.379968:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776472.379969:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776472.379970:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776472.379972:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776472.379974:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776472.379976:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776472.379978:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776472.379980:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776472.379982:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710417024, portal 10 00000100:00000001:0.0:1775776472.379983:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776472.379984:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004928 : -123823860546688 : ffff8f6202c70380) 00000100:00000040:0.0:1775776472.379988:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70380 x1862022710417024/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776497 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.379993:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776472.379995:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776472.379996:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776472.379997:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710417024, offset 0 00000400:00000200:0.0:1775776472.379999:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776472.380002:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776472.380004:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776472.380006:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776472.380007:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776472.380009:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776472.380010:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776472.380013:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453810:453810:256:4294967295] 192.168.201.115@tcp LPNI seq info [453735:453735:8:4294967295] 00000400:00000200:0.0:1775776472.380015:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776472.380024:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776472.380029:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776472.380032:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776472.380035:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776472.380037:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775776472.380041:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776472.380043:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1775776472.380046:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776472.380047:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776472.380049:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776472.380053:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776472.380055:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776472.380057:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776472.380059:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776472.380061:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70380 x1862022710417024/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776497 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.380068:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776472.380071:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70380 x1862022710417024/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776497 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.380081:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776472.380083:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776472.380086:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776472.380089:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776472.380092:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776472.380094:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308027403 00000100:00000001:0.0:1775776472.380096:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308031238 00000100:00000001:0.0:1775776472.380097:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308027403 00000100:00000001:0.0:1775776472.380099:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308031238 00000100:00000001:0.0:1775776472.380101:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776472.380102:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776472.380104:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776472.380107:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776472/real 1775776472] req@ffff8f6202c70380 x1862022710417024/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776472.380113:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776472.380114:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776472.380116:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776472.380118:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776472.380119:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776472.380121:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776472.380123:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70380 x1862022710417024/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776472.380129:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70380 x1862022710417024/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.380153:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776472.380154:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776472.380156:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776472.380157:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776472.380161:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70380 x1862022710417024/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.380167:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776472.380170:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70380 x1862022710417024/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.380176:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776472.380178:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776472.380180:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776472.380181:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776472.380182:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776472.380187:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776472.380189:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776472.380192:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776472.380194:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776472.380198:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70380 x1862022710417024/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776472.380203:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710417024:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776472.380206:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776472.380208:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73480 x1862022710417152/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.380212:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776472.380212:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776472.380214:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776472.380215:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776472.380216:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776472.380217:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710417152:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776472.380219:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776472.380220:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776472.380222:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776472.380223:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776472.380225:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776472.380226:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776472.380229:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776472.380230:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776472.380232:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776472.380234:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710417152, portal 10 00000100:00000001:0.0:1775776472.380235:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776472.380236:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849017472 : -123823860534144 : ffff8f6202c73480) 00000100:00000040:0.0:1775776472.380239:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73480 x1862022710417152/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776497 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.380243:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776472.380244:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776472.380246:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776472.380247:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710417152, offset 0 00000400:00000200:0.0:1775776472.380249:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776472.380251:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776472.380254:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776472.380255:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776472.380257:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776472.380258:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776472.380260:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776472.380262:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453811:453811:256:4294967295] 192.168.201.115@tcp LPNI seq info [453736:453736:8:4294967295] 00000400:00000200:0.0:1775776472.380264:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776472.380268:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776472.380270:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776472.380272:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776472.380274:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776472.380276:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1775776472.380278:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776472.380279:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1775776472.380282:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776472.380283:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776472.380285:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776472.380287:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776472.380289:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776472.380290:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776472.380291:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776472.380293:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73480 x1862022710417152/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776497 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.380297:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776472.380299:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73480 x1862022710417152/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776497 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.380303:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776472.380304:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776472.380305:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776472.380307:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776472.380308:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776472.380310:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308027404 00000100:00000001:0.0:1775776472.380311:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308031238 00000100:00000001:0.0:1775776472.380312:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308027404 00000100:00000001:0.0:1775776472.380314:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308031238 00000100:00000001:0.0:1775776472.380315:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776472.380316:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776472.380317:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776472.380319:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776472/real 1775776472] req@ffff8f6202c73480 x1862022710417152/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776472.380323:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776472.380324:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776472.380324:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776472.380326:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776472.380327:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776472.380328:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776472.380330:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73480 x1862022710417152/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776472.380335:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73480 x1862022710417152/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.380338:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776472.380339:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776472.380340:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776472.380340:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776472.380342:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73480 x1862022710417152/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.380345:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776472.380347:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73480 x1862022710417152/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.380351:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776472.380352:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776472.380353:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776472.380353:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776472.380354:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776472.380355:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776472.380357:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776472.380358:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776472.380360:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776472.380362:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73480 x1862022710417152/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776472.380366:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710417152:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776472.380368:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776472.380370:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776472.380371:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70380 x1862022710417024/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.380375:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776472.380376:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776472.380377:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776472.380378:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776472.380379:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776472.380379:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776472.380380:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776472.380381:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37c00. 02000000:00000001:0.0:1775776472.380383:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776472.380383:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776472.380384:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70380. 00000100:00000001:0.0:1775776472.380386:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776472.380386:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776472.380387:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776472.380389:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73480 x1862022710417152/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776497 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776472.380392:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776472.380393:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776472.380394:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776472.380395:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776472.380395:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776472.380396:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776472.380397:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776472.380397:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35000. 02000000:00000001:0.0:1775776472.380398:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776472.380399:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776472.380400:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73480. 00000100:00000001:0.0:1775776472.380401:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776472.380402:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776472.380402:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776472.380405:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776472.380405:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776472.380406:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776472.380408:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776472.507247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776472.507251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776472.507254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776472.507255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776472.507258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776472.507259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776472.507326:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776472.507331:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776472.507334:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776473.403256:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776473.403263:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776473.403266:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776473.403272:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776473.403275:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776473.403280:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776473.403283:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776473.403286:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776473.403289:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776473.531239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776473.531244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776473.531247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776473.531248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776473.531251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776473.531252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776473.531305:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776473.531309:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776473.531311:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776474.427283:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776474.427293:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776474.427295:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776474.427304:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776474.427309:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776474.427314:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776474.427317:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776474.427320:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776474.427322:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776474.555243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776474.555248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776474.555253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776474.555254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776474.555257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776474.555258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776474.555312:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776474.555315:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776474.555318:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776475.451270:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776475.451277:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776475.451281:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776475.451286:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776475.451290:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776475.451294:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776475.451298:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776475.451305:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776475.451308:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776475.579264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776475.579271:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776475.579272:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776475.579275:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776475.579279:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776475.579281:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776475.579282:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776475.579284:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776475.579287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776475.579288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776475.579290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776475.579291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776475.579360:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776475.579364:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776475.579366:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776476.283237:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308031238, at 4308031296 00000100:00000010:1.0:1775776476.283251:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c37e0. 00000100:00000040:0.0:1775776476.283257:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308031238, at 4308031296 00000100:00000010:0.0:1775776476.283265:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776476.475240:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776476.475245:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776476.475247:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776476.475251:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776476.475254:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776476.475257:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776476.475260:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776476.475261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776476.475263:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776476.603237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776476.603242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776476.603246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776476.603247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776476.603249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776476.603250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776476.603326:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776476.603330:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776476.603334:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776477.499241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776477.499247:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776477.499248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776477.499253:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776477.499257:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776477.499260:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776477.499262:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776477.499264:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776477.499266:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776477.499273:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776477.499280:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776477.499283:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776477.499287:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776477.499292:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776477.499299:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13360 00000400:00000200:1.0:1775776477.499309:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776477.499315:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776477.499318:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776477.499322:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776477.499325:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776477.499328:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776477.499331:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:1.0:1775776477.499335:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776477.499337:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:1.0:1775776477.499340:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776477.499342:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:1.0:1775776477.499345:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:1.0:1775776477.499347:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:1.0:1775776477.499351:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776477.499353:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776477.499355:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776477.499357:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:1.0:1775776477.499360:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:1.0:1775776477.499364:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776477.499367:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776477.499371:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776477.499374:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776477.499376:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776477.499378:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776477.499379:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776477.499387:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623491aa00. 00000020:00000040:1.0:1775776477.499390:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:1.0:1775776477.499393:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776477.499394:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776477.499396:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776477.499398:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776477.499402:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776477.499408:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4c400. 00000100:00000001:1.0:1775776477.499422:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776477.499432:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623491aa00 x1862022710417280/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776477.499443:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776477.499448:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776477.499454:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fa7ee57c] to pc [ptlrpcd_rcv+-1] req@ffff8f623491aa00 x1862022710417280/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776477.499475:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776477.499478:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776477.499482:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776477.499483:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776477.499485:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776477.499488:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776477.499491:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13360 00000400:00000200:1.0:1775776477.499496:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776477.499500:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776477.499503:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776477.499507:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776477.499510:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776477.499513:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776477.499514:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776477.499516:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:0.0:1775776477.499520:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1775776477.499520:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776477.499521:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:0.0:1775776477.499524:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1775776477.499525:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776477.499529:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:1.0:1775776477.499531:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:1.0:1775776477.499535:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775776477.499536:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491aa00 x1862022710417280/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776477.499541:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776477.499546:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776477.499552:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776477.499557:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:0.0:1775776477.499560:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:1.0:1775776477.499562:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000001:0.0:1775776477.499563:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776477.499568:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000001:0.0:1775776477.499570:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1775776477.499572:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776477.499576:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776477.499582:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776477.499583:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776477.499585:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776477.499591:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775776477.499592:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1775776477.499596:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775776477.499599:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710417280:192.168.201.115@tcp:250:kworker.0 00000008:00000001:1.0:1775776477.499601:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776477.499606:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776477.499608:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000010:1.0:1775776477.499612:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6234918e00. 02000000:00000001:0.0:1775776477.499613:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:1.0:1775776477.499616:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776477.499618:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776477.499621:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776477.499623:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1775776477.499623:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776477.499627:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776477.499629:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:0.0:1775776477.499631:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7800. 02000000:00000001:1.0:1775776477.499631:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776477.499637:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776477.499643:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776477.499649:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 02000000:00000010:1.0:1775776477.499653:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4d000. 00000400:00000010:0.0:1775776477.499656:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000001:1.0:1775776477.499659:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775776477.499665:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710417280, portal 25 00000100:00100000:1.0:1775776477.499668:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6234918e00 x1862022710417408/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776477.499670:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776477.499674:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684369408 : -123823025182208 : ffff8f623491aa00) 00000100:00000001:1.0:1775776477.499686:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:0.0:1775776477.499687:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491aa00 x1862022710417280/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776502 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776477.499690:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776477.499699:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000416b2640] to pc [ptlrpcd_rcv+-1] req@ffff8f6234918e00 x1862022710417408/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.499705:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776477.499714:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776477.499721:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000001:1.0:1775776477.499721:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776477.499727:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:0.0:1775776477.499728:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710417280, offset 0 00000100:00080000:1.0:1775776477.499735:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776477.499737:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776477.499740:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1775776477.499741:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:1.0:1775776477.499745:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776477.499750:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13360 00000400:00000200:0.0:1775776477.499754:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776477.499763:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776477.499769:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776477.499773:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776477.499777:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776477.499780:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776477.499784:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453812:453812:256:4294967295] 192.168.201.115@tcp LPNI seq info [453737:453737:8:4294967295] 00000400:00000200:0.0:1775776477.499790:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776477.499800:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000400:00000200:1.0:1775776477.499810:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000200:0.0:1775776477.499811:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776477.499818:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000800:00000010:0.0:1775776477.499820:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000100:00080000:1.0:1775776477.499836:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776477.499840:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776477.499842:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776477.499846:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:0.0:1775776477.499847:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:1.0:1775776477.499848:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:1.0:1775776477.499853:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776477.499854:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000800:00000200:0.0:1775776477.499855:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000020:00000001:1.0:1775776477.499856:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776477.499859:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:1.0:1775776477.499862:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:1.0:1775776477.499864:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:1.0:1775776477.499869:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000800:00000200:0.1:1775776477.499870:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:1.0:1775776477.499871:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776477.499875:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776477.499878:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000200:0.1:1775776477.499879:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000020:00000040:1.0:1775776477.499881:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:1.0:1775776477.499885:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776477.499887:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776477.499890:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:0.0:1775776477.499892:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 02000000:00000001:1.0:1775776477.499894:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776477.499897:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775776477.499898:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000100:00000001:1.0:1775776477.499899:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776477.499902:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776477.499905:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776477.499906:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776477.499907:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000010:1.0:1775776477.499909:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6234918380. 00000020:00000040:1.0:1775776477.499913:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:0.0:1775776477.499917:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 02000000:00000001:1.0:1775776477.499918:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776477.499922:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776477.499925:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1775776477.499926:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 02000000:00000001:1.0:1775776477.499927:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:0.0:1775776477.499932:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 02000000:00000001:1.0:1775776477.499932:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776477.499939:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4dc00. 00000100:00000001:0.0:1775776477.499940:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1775776477.499946:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775776477.499948:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491aa00 x1862022710417280/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776502 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776477.499955:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6234918380 x1862022710417536/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776477.499970:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1775776477.499972:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776477.499976:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776477.499979:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491aa00 x1862022710417280/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776502 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776477.499984:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072d9a890] to pc [ptlrpcd_rcv+-1] req@ffff8f6234918380 x1862022710417536/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500004:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776477.500006:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776477.500007:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1775776477.500009:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1775776477.500013:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1775776477.500015:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776477.500015:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1775776477.500021:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:0.0:1775776477.500024:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:1.0:1775776477.500025:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13370) 00000100:00000001:0.0:1775776477.500031:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308036358 00000100:00000010:0.0:1775776477.500038:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776477.500046:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776477.500052:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308036358 00000100:00000010:0.0:1775776477.500056:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c38a0. 00000100:00000001:0.0:1775776477.500059:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776477.500063:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776477.500066:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776477.500068:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776477.500073:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776477/real 1775776477] req@ffff8f623491aa00 x1862022710417280/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776477.500084:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776477.500086:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776477.500088:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776477.500091:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776477.500093:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776477.500095:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776477.500099:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491aa00 x1862022710417280/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776477.500109:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491aa00 x1862022710417280/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500117:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776477.500119:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776477.500122:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776477.500124:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776477.500128:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491aa00 x1862022710417280/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500158:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776477.500163:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491aa00 x1862022710417280/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500171:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776477.500174:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776477.500176:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776477.500177:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776477.500179:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776477.500182:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776477.500185:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776477.500189:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776477.500191:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776477.500196:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491aa00 x1862022710417280/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776477.500204:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710417280:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776477.500209:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776477.500211:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776477.500214:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491aa00 x1862022710417280/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500222:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776477.500224:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776477.500227:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776477.500230:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776477.500232:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776477.500233:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776477.500235:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776477.500238:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4c400. 02000000:00000001:0.0:1775776477.500241:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776477.500243:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776477.500245:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491aa00. 00000100:00000001:0.0:1775776477.500249:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776477.500250:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776477.500252:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776477.500259:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776477.500260:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776477.500263:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776477.500264:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776477.500266:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776477.500269:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918e00 x1862022710417408/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500277:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776477.500279:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776477.500281:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776477.500283:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776477.500285:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776477.500289:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710417408:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776477.500292:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776477.500294:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776477.500296:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776477.500298:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776477.500301:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776477.500303:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776477.500313:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776477.500316:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776477.500320:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776477.500324:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710417408, portal 10 00000100:00000001:0.0:1775776477.500326:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776477.500327:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684362240 : -123823025189376 : ffff8f6234918e00) 00000100:00000040:0.0:1775776477.500335:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918e00 x1862022710417408/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776502 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500342:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776477.500345:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776477.500348:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776477.500350:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710417408, offset 0 00000400:00000200:0.0:1775776477.500355:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776477.500359:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776477.500363:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776477.500366:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776477.500369:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776477.500372:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776477.500374:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776477.500377:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453813:453813:256:4294967295] 192.168.201.115@tcp LPNI seq info [453738:453738:8:4294967295] 00000400:00000200:0.0:1775776477.500381:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776477.500387:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776477.500392:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776477.500395:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776477.500399:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776477.500402:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775776477.500407:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776477.500410:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1775776477.500414:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776477.500416:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776477.500419:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776477.500423:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776477.500426:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776477.500428:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776477.500430:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776477.500434:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918e00 x1862022710417408/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776502 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500442:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776477.500445:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918e00 x1862022710417408/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776502 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500452:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776477.500454:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776477.500457:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776477.500462:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776477.500465:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776477.500467:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308032524 00000100:00000001:0.0:1775776477.500470:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308036358 00000100:00000001:0.0:1775776477.500472:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308032524 00000100:00000001:0.0:1775776477.500474:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308036358 00000100:00000001:0.0:1775776477.500476:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776477.500477:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776477.500479:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776477.500483:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776477/real 1775776477] req@ffff8f6234918e00 x1862022710417408/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776477.500490:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776477.500491:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776477.500493:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776477.500495:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776477.500496:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776477.500498:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776477.500501:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918e00 x1862022710417408/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776477.500509:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918e00 x1862022710417408/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500515:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776477.500516:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776477.500518:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776477.500520:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776477.500523:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918e00 x1862022710417408/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500530:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776477.500534:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918e00 x1862022710417408/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500542:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776477.500545:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776477.500546:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776477.500548:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776477.500550:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776477.500555:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776477.500557:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776477.500561:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776477.500564:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776477.500567:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918e00 x1862022710417408/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776477.500575:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710417408:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776477.500579:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776477.500582:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234918380 x1862022710417536/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500589:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776477.500591:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776477.500593:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776477.500595:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776477.500597:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776477.500600:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234918380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710417536:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776477.500604:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776477.500606:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776477.500608:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776477.500609:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776477.500613:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776477.500615:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776477.500619:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776477.500621:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776477.500624:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776477.500627:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710417536, portal 10 00000100:00000001:0.0:1775776477.500629:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776477.500630:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684359552 : -123823025192064 : ffff8f6234918380) 00000100:00000040:0.0:1775776477.500635:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234918380 x1862022710417536/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776502 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500642:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776477.500644:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776477.500647:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776477.500649:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710417536, offset 0 00000400:00000200:0.0:1775776477.500654:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776477.500658:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776477.500661:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776477.500664:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776477.500667:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776477.500670:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776477.500673:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776477.500676:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453814:453814:256:4294967295] 192.168.201.115@tcp LPNI seq info [453739:453739:8:4294967295] 00000400:00000200:0.0:1775776477.500681:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776477.500687:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776477.500692:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776477.500696:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776477.500700:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776477.500703:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1775776477.500707:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776477.500709:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1775776477.500714:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776477.500716:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776477.500719:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776477.500722:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776477.500725:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776477.500727:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776477.500728:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776477.500731:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234918380 x1862022710417536/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776502 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500737:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776477.500739:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234918380 x1862022710417536/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776502 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500743:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776477.500744:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776477.500747:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776477.500749:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776477.500751:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776477.500753:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308032524 00000100:00000001:0.0:1775776477.500754:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308036358 00000100:00000001:0.0:1775776477.500756:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308032524 00000100:00000001:0.0:1775776477.500758:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308036358 00000100:00000001:0.0:1775776477.500759:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776477.500760:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776477.500761:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776477.500763:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776477/real 1775776477] req@ffff8f6234918380 x1862022710417536/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776477.500768:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776477.500769:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776477.500771:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776477.500772:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776477.500773:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776477.500775:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776477.500776:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234918380 x1862022710417536/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776477.500782:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234918380 x1862022710417536/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500786:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776477.500787:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776477.500789:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776477.500790:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776477.500792:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234918380 x1862022710417536/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500796:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776477.500798:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234918380 x1862022710417536/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500802:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776477.500803:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776477.500804:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776477.500805:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776477.500806:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776477.500807:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776477.500809:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776477.500811:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776477.500812:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776477.500814:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234918380 x1862022710417536/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776477.500819:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234918380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710417536:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776477.500821:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776477.500822:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776477.500841:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918e00 x1862022710417408/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500846:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776477.500848:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776477.500849:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776477.500851:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776477.500852:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776477.500853:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776477.500854:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776477.500856:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4d000. 02000000:00000001:0.0:1775776477.500857:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776477.500858:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776477.500859:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918e00. 00000100:00000001:0.0:1775776477.500861:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776477.500862:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776477.500863:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776477.500865:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234918380 x1862022710417536/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776502 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776477.500869:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776477.500870:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776477.500871:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776477.500872:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776477.500873:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776477.500873:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776477.500874:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776477.500875:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4dc00. 02000000:00000001:0.0:1775776477.500876:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776477.500877:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776477.500878:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234918380. 00000100:00000001:0.0:1775776477.500879:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776477.500880:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776477.500881:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776477.500883:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776477.500884:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776477.500885:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776477.500886:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776477.627246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776477.627250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776477.627255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776477.627256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776477.627258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776477.627259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776477.627325:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776477.627329:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776477.627332:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776478.523275:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776478.523283:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776478.523285:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776478.523291:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776478.523297:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776478.523301:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776478.523304:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776478.523306:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776478.523309:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776478.651229:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776478.651233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776478.651237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776478.651238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776478.651241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776478.651242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776478.651318:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776478.651321:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776478.651324:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776479.547203:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776479.547209:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776479.547211:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776479.547215:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776479.547217:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776479.547220:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776479.547222:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776479.547224:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776479.547227:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776479.675246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776479.675251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776479.675255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776479.675256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776479.675258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776479.675259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776479.675321:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776479.675324:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776479.675327:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776480.571237:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776480.571244:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776480.571247:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776480.571252:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776480.571256:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776480.571261:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776480.571263:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776480.571266:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776480.571269:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776480.699229:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776480.699236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776480.699241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776480.699243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776480.699246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776480.699247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776480.699308:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776480.699311:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776480.699314:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776481.403234:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308036358, at 4308036416 00000100:00000010:1.0:1775776481.403246:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c38a0. 00000100:00000040:0.0:1775776481.403250:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308036358, at 4308036416 00000100:00000010:0.0:1775776481.403259:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3480. 00000100:00000001:0.0:1775776481.595221:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776481.595227:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776481.595237:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776481.595242:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776481.595245:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776481.595248:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776481.595250:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776481.595252:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776481.595254:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776481.723254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776481.723259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776481.723263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776481.723265:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776481.723267:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776481.723269:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776481.723271:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776481.723273:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776481.723274:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776481.723276:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776481.723279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776481.723279:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776481.723280:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776481.723281:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776481.723282:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776481.723283:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776481.723284:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776481.723285:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776481.723356:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776481.723360:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776481.723364:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776482.619236:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776482.619239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776482.619244:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1775776482.619247:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776482.619247:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776482.619252:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1775776482.619252:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:0.0:1775776482.619257:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776482.619257:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776482.619259:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776482.619262:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776482.619263:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776482.619266:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776482.619268:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776482.619271:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13365 00000400:00000200:0.0:1775776482.619281:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776482.619290:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776482.619293:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776482.619295:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776482.619298:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776482.619300:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776482.619302:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776482.619304:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776482.619306:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776482.619308:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776482.619310:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776482.619312:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776482.619313:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776482.619315:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776482.619316:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776482.619317:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776482.619318:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776482.619320:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776482.619322:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776482.619324:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776482.619326:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776482.619329:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776482.619331:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776482.619341:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776482.619343:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776482.619354:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060e00. 00000020:00000040:0.0:1775776482.619357:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776482.619361:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776482.619362:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776482.619364:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776482.619367:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776482.619370:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776482.619376:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7800. 00000100:00000001:0.0:1775776482.619381:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776482.619388:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060e00 x1862022710417664/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776482.619396:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776482.619400:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776482.619403:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004a3bddaa] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060e00 x1862022710417664/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776482.619419:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776482.619421:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776482.619424:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776482.619425:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776482.619426:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776482.619428:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776482.619430:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13365 00000400:00000200:0.0:1775776482.619433:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776482.619436:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776482.619438:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776482.619440:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776482.619442:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776482.619443:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1775776482.619444:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776482.619447:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:1.0:1775776482.619448:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1775776482.619451:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775776482.619452:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:0.0:1775776482.619453:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776482.619456:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776482.619458:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000040:1.0:1775776482.619459:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e060e00 x1862022710417664/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775776482.619460:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775776482.619464:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775776482.619470:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776482.619474:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776482.619476:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1775776482.619477:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776482.619478:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776482.619479:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776482.619482:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:1.0:1775776482.619483:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1775776482.619485:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000001:1.0:1775776482.619486:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776482.619487:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776482.619490:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776482.619490:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776482.619496:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e060e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710417664:192.168.201.115@tcp:250:kworker.0 02000000:00000001:0.0:1775776482.619497:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776482.619502:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776482.619502:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1775776482.619505:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776482.619507:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:0.0:1775776482.619509:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776482.619511:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776482.619511:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776482.619515:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776482.619518:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:0.0:1775776482.619519:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063100. 00000020:00000040:0.0:1775776482.619522:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776482.619524:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776482.619525:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776482.619526:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776482.619527:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:1.0:1775776482.619527:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:0.0:1775776482.619529:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776482.619533:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776482.619535:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6000. 00000100:00000001:0.0:1775776482.619538:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776482.619543:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00100000:0.0:1775776482.619544:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063100 x1862022710417792/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776482.619551:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1775776482.619551:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000001:0.0:1775776482.619554:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776482.619558:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033f6db7f] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063100 x1862022710417792/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1775776482.619562:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000001:0.0:1775776482.619564:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776482.619567:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776482.619569:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776482.619570:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776482.619571:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1775776482.619572:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710417664, portal 25 00000100:00000001:0.0:1775776482.619573:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776482.619577:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13365 00000100:00000001:1.0:1775776482.619578:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000400:00000200:0.0:1775776482.619581:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776482.619582:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574556672 : -123823134994944 : ffff8f622e060e00) 00000400:00000200:0.0:1775776482.619584:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776482.619588:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776482.619591:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776482.619593:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776482.619597:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000040:1.0:1775776482.619597:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e060e00 x1862022710417664/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776507 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776482.619600:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776482.619604:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776482.619606:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776482.619608:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776482.619610:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775776482.619614:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775776482.619617:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775776482.619622:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776482.619623:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776482.619624:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776482.619626:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776482.619627:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775776482.619630:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776482.619631:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1775776482.619632:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776482.619635:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1775776482.619638:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 02000000:00000001:0.0:1775776482.619639:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776482.619642:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776482.619644:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000010:1.0:1775776482.619644:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000008:00100000:0.0:1775776482.619646:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776482.619649:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776482.619649:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710417664, offset 0 00000100:00000001:0.0:1775776482.619650:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776482.619657:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060700. 00000020:00000040:0.0:1775776482.619661:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:1.0:1775776482.619661:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 02000000:00000001:0.0:1775776482.619665:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776482.619667:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776482.619668:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776482.619669:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776482.619670:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776482.619674:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000010:0.0:1775776482.619676:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4800. 00000100:00000001:0.0:1775776482.619679:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776482.619681:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00100000:0.0:1775776482.619686:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060700 x1862022710417920/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776482.619688:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776482.619695:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:0.0:1775776482.619697:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776482.619700:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776482.619701:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000040:0.0:1775776482.619705:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000225c8695] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060700 x1862022710417920/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776482.619707:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776482.619715:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453815:453815:256:4294967295] 192.168.201.115@tcp LPNI seq info [453740:453740:8:4294967295] 00000100:00000001:0.0:1775776482.619723:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776482.619723:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:0.0:1775776482.619728:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776482.619745:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776482.619751:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776482.619753:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000040:0.0:1775776482.619759:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.0:1775776482.619765:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000040:0.0:1775776482.619767:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13375) 00000800:00000010:1.0:1775776482.619772:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776482.619781:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776482.619786:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775776482.619793:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776482.619797:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775776482.619802:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776482.619804:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776482.619811:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776482.619815:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776482.619819:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776482.619821:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776482.619826:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776482.619830:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e060e00 x1862022710417664/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776507 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.619836:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776482.619839:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e060e00 x1862022710417664/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776507 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.619844:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776482.619845:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776482.619849:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776482.619852:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776482.619855:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308041478 00000100:00000010:1.0:1775776482.619860:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e28a0. 00000100:00000001:1.0:1775776482.619866:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776482.619872:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308041478 00000100:00000010:1.0:1775776482.619875:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2e40. 00000100:00000001:1.0:1775776482.619878:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776482.619881:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776482.619884:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776482.619886:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776482.619891:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776482/real 1775776482] req@ffff8f622e060e00 x1862022710417664/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776482.619901:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776482.619903:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776482.619905:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776482.619908:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776482.619910:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776482.619912:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776482.619916:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e060e00 x1862022710417664/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776482.619923:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e060e00 x1862022710417664/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.619928:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776482.619929:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776482.619931:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776482.619932:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776482.619934:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e060e00 x1862022710417664/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.619939:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776482.619942:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e060e00 x1862022710417664/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.619947:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776482.619949:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776482.619950:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776482.619950:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776482.619952:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776482.619953:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776482.619955:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776482.619958:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776482.619959:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776482.619962:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e060e00 x1862022710417664/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776482.619967:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e060e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710417664:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776482.619969:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776482.619971:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776482.619973:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e060e00 x1862022710417664/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.619982:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776482.619984:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776482.619987:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776482.619989:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776482.619991:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776482.619992:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776482.619993:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776482.619995:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:1.0:1775776482.619998:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776482.619999:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776482.620001:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e060e00. 00000100:00000001:1.0:1775776482.620003:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776482.620004:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776482.620006:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776482.620011:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776482.620012:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776482.620014:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776482.620015:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776482.620016:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776482.620018:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e063100 x1862022710417792/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.620023:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776482.620024:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776482.620026:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776482.620027:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776482.620029:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776482.620031:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e063100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710417792:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776482.620032:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776482.620034:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776482.620035:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776482.620036:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776482.620038:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776482.620040:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776482.620043:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776482.620045:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776482.620048:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776482.620050:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710417792, portal 10 00000100:00000001:1.0:1775776482.620052:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776482.620053:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574565632 : -123823134985984 : ffff8f622e063100) 00000100:00000040:1.0:1775776482.620057:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e063100 x1862022710417792/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776507 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.620062:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776482.620064:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776482.620065:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776482.620067:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710417792, offset 0 00000400:00000200:1.0:1775776482.620069:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776482.620072:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776482.620075:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776482.620077:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776482.620079:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776482.620081:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776482.620082:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776482.620085:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453816:453816:256:4294967295] 192.168.201.115@tcp LPNI seq info [453741:453741:8:4294967295] 00000400:00000200:1.0:1775776482.620087:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776482.620092:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776482.620095:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776482.620097:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776482.620100:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776482.620102:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775776482.620105:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776482.620107:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775776482.620110:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776482.620111:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776482.620113:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776482.620116:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776482.620118:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776482.620119:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776482.620122:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776482.620124:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e063100 x1862022710417792/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776507 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.620130:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776482.620131:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e063100 x1862022710417792/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776507 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.620156:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776482.620158:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776482.620161:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776482.620164:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776482.620167:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776482.620168:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308037644 00000100:00000001:1.0:1775776482.620171:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308041478 00000100:00000001:1.0:1775776482.620173:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308037644 00000100:00000001:1.0:1775776482.620174:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308041478 00000100:00000001:1.0:1775776482.620175:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776482.620177:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776482.620178:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776482.620181:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776482/real 1775776482] req@ffff8f622e063100 x1862022710417792/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776482.620186:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776482.620187:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776482.620188:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776482.620190:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776482.620191:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776482.620192:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776482.620194:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e063100 x1862022710417792/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776482.620200:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e063100 x1862022710417792/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.620208:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776482.620209:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776482.620210:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776482.620211:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776482.620213:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e063100 x1862022710417792/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.620217:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776482.620220:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e063100 x1862022710417792/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.620224:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776482.620225:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776482.620226:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776482.620227:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776482.620228:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776482.620231:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776482.620233:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776482.620235:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776482.620236:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776482.620238:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e063100 x1862022710417792/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776482.620243:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e063100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710417792:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776482.620245:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776482.620247:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e060700 x1862022710417920/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.620252:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776482.620253:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776482.620255:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776482.620256:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776482.620257:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776482.620259:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e060700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710417920:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776482.620261:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776482.620262:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776482.620263:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776482.620264:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776482.620267:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776482.620268:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776482.620270:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776482.620271:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776482.620273:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776482.620275:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710417920, portal 10 00000100:00000001:1.0:1775776482.620277:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776482.620278:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574554880 : -123823134996736 : ffff8f622e060700) 00000100:00000040:1.0:1775776482.620281:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e060700 x1862022710417920/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776507 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.620286:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776482.620287:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776482.620288:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776482.620290:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710417920, offset 0 00000400:00000200:1.0:1775776482.620292:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776482.620295:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776482.620297:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776482.620299:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776482.620301:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776482.620302:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776482.620304:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776482.620306:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453817:453817:256:4294967295] 192.168.201.115@tcp LPNI seq info [453742:453742:8:4294967295] 00000400:00000200:1.0:1775776482.620309:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776482.620312:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776482.620316:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776482.620318:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776482.620320:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776482.620322:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775776482.620325:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776482.620326:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775776482.620329:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776482.620330:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776482.620332:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776482.620335:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776482.620337:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776482.620338:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776482.620339:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776482.620341:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e060700 x1862022710417920/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776507 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.620346:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776482.620348:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e060700 x1862022710417920/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776507 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.620352:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776482.620353:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776482.620355:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776482.620356:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776482.620358:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776482.620359:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308037644 00000100:00000001:1.0:1775776482.620361:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308041478 00000100:00000001:1.0:1775776482.620362:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308037644 00000100:00000001:1.0:1775776482.620364:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308041478 00000100:00000001:1.0:1775776482.620365:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776482.620366:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776482.620368:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776482.620370:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776482/real 1775776482] req@ffff8f622e060700 x1862022710417920/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776482.620375:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776482.620376:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776482.620377:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776482.620379:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776482.620380:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776482.620381:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776482.620382:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e060700 x1862022710417920/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776482.620387:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e060700 x1862022710417920/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.620391:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776482.620392:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776482.620393:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776482.620394:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776482.620396:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e060700 x1862022710417920/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.620400:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776482.620402:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e060700 x1862022710417920/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.620406:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776482.620408:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776482.620409:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776482.620410:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776482.620411:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776482.620412:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776482.620413:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776482.620415:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776482.620416:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776482.620419:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e060700 x1862022710417920/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776482.620424:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e060700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710417920:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776482.620426:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776482.620427:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776482.620429:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e063100 x1862022710417792/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.620433:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776482.620434:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776482.620435:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776482.620436:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776482.620437:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776482.620438:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776482.620439:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776482.620440:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:1.0:1775776482.620442:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776482.620442:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776482.620443:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e063100. 00000100:00000001:1.0:1775776482.620445:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776482.620446:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776482.620447:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776482.620449:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e060700 x1862022710417920/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776507 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776482.620453:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776482.620454:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776482.620455:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776482.620456:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776482.620456:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776482.620457:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776482.620458:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776482.620459:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:1.0:1775776482.620460:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776482.620461:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776482.620462:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e060700. 00000100:00000001:1.0:1775776482.620464:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776482.620465:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776482.620466:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776482.620468:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776482.620469:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776482.620470:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776482.620471:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776482.747215:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776482.747221:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776482.747226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776482.747227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776482.747231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776482.747233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776482.747315:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776482.747319:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776482.747323:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776483.643242:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776483.643247:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776483.643248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776483.643253:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776483.643256:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776483.643258:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776483.643261:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776483.643263:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776483.643264:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776483.771278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776483.771285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776483.771291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776483.771293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776483.771297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776483.771298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776483.771367:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776483.771371:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776483.771375:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776484.667239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776484.667245:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776484.667248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776484.667253:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776484.667257:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776484.667261:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776484.667264:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776484.667266:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776484.667269:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776484.795241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776484.795247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776484.795253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776484.795255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776484.795259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776484.795260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776484.795315:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776484.795319:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776484.795322:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776485.691263:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776485.691271:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776485.691273:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776485.691278:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776485.691282:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776485.691286:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776485.691289:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776485.691291:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776485.691295:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776485.819228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776485.819234:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776485.819236:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776485.819239:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776485.819243:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776485.819244:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776485.819246:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776485.819247:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776485.819250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776485.819251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776485.819255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776485.819256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776485.819327:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776485.819331:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776485.819334:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776486.523256:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308041478, at 4308041536 00000100:00000040:0.0:1775776486.523257:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308041478, at 4308041536 00000100:00000010:0.0:1775776486.523269:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e28a0. 00000100:00000010:1.0:1775776486.523272:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2e40. 00000100:00000001:1.0:1775776486.715227:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776486.715235:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776486.715237:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776486.715244:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776486.715248:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776486.715252:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776486.715254:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776486.715257:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776486.715260:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776486.843248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776486.843254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776486.843260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776486.843262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776486.843266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776486.843268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776486.843340:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776486.843344:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776486.843348:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776487.739254:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776487.739257:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776487.739263:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1775776487.739265:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776487.739267:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776487.739269:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776487.739274:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775776487.739274:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776487.739280:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776487.739282:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776487.739285:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776487.739289:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775776487.739291:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13370 00000100:00000001:1.0:1775776487.739293:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776487.739298:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776487.739305:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776487.739312:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776487.739316:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776487.739320:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776487.739323:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776487.739325:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776487.739327:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776487.739330:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776487.739332:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776487.739334:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776487.739336:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776487.739338:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776487.739340:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776487.739342:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776487.739343:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776487.739345:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776487.739346:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776487.739348:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776487.739350:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776487.739351:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776487.739354:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776487.739357:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776487.739359:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776487.739361:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776487.739362:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776487.739370:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061880. 00000020:00000040:0.0:1775776487.739373:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776487.739377:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776487.739378:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776487.739380:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776487.739382:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776487.739386:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776487.739391:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5800. 00000100:00000001:0.0:1775776487.739397:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776487.739403:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061880 x1862022710418048/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776487.739411:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776487.739414:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776487.739417:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005afd1f27] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061880 x1862022710418048/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776487.739436:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776487.739438:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776487.739441:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776487.739442:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776487.739443:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776487.739445:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776487.739446:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13370 00000400:00000200:0.0:1775776487.739449:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776487.739451:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776487.739453:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776487.739455:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776487.739457:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776487.739460:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776487.739462:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776487.739464:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776487.739465:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776487.739467:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776487.739469:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775776487.739470:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775776487.739472:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776487.739472:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1775776487.739475:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776487.739477:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776487.739478:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776487.739481:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776487.739482:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776487.739486:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776487.739487:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775776487.739492:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000040:1.0:1775776487.739492:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061880 x1862022710418048/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776487.739494:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776487.739497:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776487.739503:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776487.739509:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1775776487.739510:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776487.739512:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776487.739514:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776487.739517:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:0.0:1775776487.739520:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775776487.739521:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1775776487.739527:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776487.739527:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776487.739531:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776487.739534:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710418048:192.168.201.115@tcp:250:kworker.0 00000100:00000010:0.0:1775776487.739537:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061c00. 00000020:00000040:0.0:1775776487.739542:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000001:1.0:1775776487.739546:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776487.739548:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776487.739550:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776487.739551:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1775776487.739555:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776487.739555:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776487.739557:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776487.739560:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1775776487.739562:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776487.739567:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 02000000:00000010:0.0:1775776487.739569:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6c00. 02000000:00000001:1.0:1775776487.739570:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776487.739573:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776487.739580:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00100000:0.0:1775776487.739581:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061c00 x1862022710418176/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1775776487.739587:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000001:0.0:1775776487.739596:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1775776487.739597:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000001:0.0:1775776487.739598:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776487.739603:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076a44879] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061c00 x1862022710418176/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776487.739607:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710418048, portal 25 00000100:00000001:1.0:1775776487.739613:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776487.739614:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776487.739616:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574559360 : -123823134992256 : ffff8f622e061880) 00000100:00080000:0.0:1775776487.739618:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776487.739621:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776487.739623:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776487.739625:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1775776487.739626:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061880 x1862022710418048/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776512 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776487.739628:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776487.739633:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13370 00000400:00000200:0.0:1775776487.739642:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776487.739659:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776487.739664:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776487.739667:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000400:00000200:0.0:1775776487.739672:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000200:1.0:1775776487.739673:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710418048, offset 0 00000100:00080000:0.0:1775776487.739674:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776487.739679:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776487.739681:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776487.739684:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776487.739686:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:0.0:1775776487.739687:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776487.739690:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776487.739691:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776487.739692:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776487.739693:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775776487.739695:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775776487.739699:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776487.739699:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:0.0:1775776487.739704:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776487.739706:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776487.739709:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000040:0.0:1775776487.739710:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776487.739714:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776487.739717:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776487.739720:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:0.0:1775776487.739722:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776487.739724:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776487.739727:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 02000000:00000001:0.0:1775776487.739728:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776487.739732:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776487.739735:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776487.739735:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1775776487.739740:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776487.739743:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1775776487.739743:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000008:00000001:0.0:1775776487.739747:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776487.739750:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776487.739752:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453818:453818:256:4294967295] 192.168.201.115@tcp LPNI seq info [453743:453743:8:4294967295] 00000100:00000010:0.0:1775776487.739754:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061180. 00000020:00000040:0.0:1775776487.739759:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776487.739764:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1775776487.739764:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:0.0:1775776487.739767:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776487.739769:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776487.739770:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776487.739772:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776487.739778:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5c00. 00000400:00000200:1.0:1775776487.739781:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:0.0:1775776487.739782:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776487.739790:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061180 x1862022710418304/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1775776487.739795:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:0.0:1775776487.739802:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776487.739804:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775776487.739804:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000100:00000040:0.0:1775776487.739811:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e23859] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061180 x1862022710418304/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1775776487.739817:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775776487.739821:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776487.739824:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.0:1775776487.739824:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000100:00000040:0.0:1775776487.739830:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776487.739833:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775776487.739837:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:0.0:1775776487.739838:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775776487.739845:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000100:00000040:0.0:1775776487.739846:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13380) 00000800:00000100:1.0:1775776487.739856:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776487.739858:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776487.739864:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776487.739870:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776487.739875:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776487.739877:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776487.739883:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776487.739888:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061880 x1862022710418048/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776512 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.739898:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776487.739902:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061880 x1862022710418048/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776512 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.739910:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776487.739913:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776487.739917:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776487.739921:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776487.739923:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308046598 00000100:00000010:1.0:1775776487.739928:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2e40. 00000100:00000001:1.0:1775776487.739934:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776487.739940:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308046598 00000100:00000010:1.0:1775776487.739943:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e27e0. 00000100:00000001:1.0:1775776487.739946:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776487.739949:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776487.739952:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776487.739954:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776487.739958:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776487/real 1775776487] req@ffff8f622e061880 x1862022710418048/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776487.739968:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776487.739970:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776487.739973:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776487.739976:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776487.739977:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776487.739979:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776487.739983:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061880 x1862022710418048/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776487.739993:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061880 x1862022710418048/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.740000:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776487.740002:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776487.740004:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776487.740006:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776487.740009:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061880 x1862022710418048/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.740017:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776487.740022:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061880 x1862022710418048/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.740029:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776487.740032:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776487.740034:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776487.740035:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776487.740038:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776487.740040:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776487.740043:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776487.740047:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776487.740049:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776487.740054:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061880 x1862022710418048/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776487.740063:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710418048:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776487.740071:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776487.740073:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776487.740076:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061880 x1862022710418048/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.740083:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776487.740086:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776487.740088:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776487.740091:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776487.740093:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776487.740094:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776487.740097:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776487.740099:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:1.0:1775776487.740103:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776487.740105:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776487.740107:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061880. 00000100:00000001:1.0:1775776487.740110:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776487.740111:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776487.740113:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776487.740120:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776487.740122:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776487.740124:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776487.740126:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776487.740127:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776487.740131:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061c00 x1862022710418176/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.740158:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776487.740159:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776487.740162:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776487.740164:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776487.740166:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776487.740169:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710418176:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776487.740173:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776487.740174:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776487.740176:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776487.740178:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776487.740181:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776487.740184:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776487.740188:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776487.740190:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776487.740193:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776487.740196:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710418176, portal 10 00000100:00000001:1.0:1775776487.740199:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776487.740200:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574560256 : -123823134991360 : ffff8f622e061c00) 00000100:00000040:1.0:1775776487.740206:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061c00 x1862022710418176/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776512 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.740214:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776487.740216:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776487.740218:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776487.740221:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710418176, offset 0 00000400:00000200:1.0:1775776487.740225:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776487.740229:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776487.740233:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776487.740236:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776487.740239:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776487.740242:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776487.740244:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776487.740248:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453819:453819:256:4294967295] 192.168.201.115@tcp LPNI seq info [453744:453744:8:4294967295] 00000400:00000200:1.0:1775776487.740253:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776487.740260:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776487.740265:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776487.740268:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776487.740272:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776487.740275:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775776487.740280:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776487.740283:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775776487.740286:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776487.740288:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776487.740292:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776487.740295:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776487.740298:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776487.740300:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776487.740303:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776487.740306:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061c00 x1862022710418176/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776512 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.740315:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776487.740318:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061c00 x1862022710418176/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776512 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.740325:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776487.740327:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776487.740329:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776487.740333:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776487.740337:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776487.740339:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308042764 00000100:00000001:1.0:1775776487.740342:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308046598 00000100:00000001:1.0:1775776487.740344:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308042764 00000100:00000001:1.0:1775776487.740346:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308046598 00000100:00000001:1.0:1775776487.740348:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776487.740350:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776487.740353:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776487.740357:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776487/real 1775776487] req@ffff8f622e061c00 x1862022710418176/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776487.740365:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776487.740367:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776487.740368:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776487.740371:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776487.740372:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776487.740375:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776487.740379:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061c00 x1862022710418176/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776487.740388:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061c00 x1862022710418176/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.740395:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776487.740397:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776487.740399:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776487.740400:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776487.740404:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061c00 x1862022710418176/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.740412:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776487.740416:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061c00 x1862022710418176/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.740423:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776487.740426:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776487.740428:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776487.740430:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776487.740432:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776487.740437:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776487.740439:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776487.740443:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776487.740446:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776487.740450:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061c00 x1862022710418176/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776487.740459:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710418176:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776487.740463:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776487.740467:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061180 x1862022710418304/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.740481:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776487.740483:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776487.740485:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776487.740487:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776487.740489:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776487.740492:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710418304:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776487.740495:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776487.740497:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776487.740499:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776487.740500:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776487.740503:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776487.740505:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776487.740508:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776487.740511:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776487.740514:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776487.740517:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710418304, portal 10 00000100:00000001:1.0:1775776487.740519:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776487.740520:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574557568 : -123823134994048 : ffff8f622e061180) 00000100:00000040:1.0:1775776487.740526:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061180 x1862022710418304/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776512 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.740534:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776487.740536:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776487.740538:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776487.740541:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710418304, offset 0 00000400:00000200:1.0:1775776487.740545:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776487.740549:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776487.740552:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776487.740555:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776487.740557:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776487.740560:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776487.740563:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776487.740567:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453820:453820:256:4294967295] 192.168.201.115@tcp LPNI seq info [453745:453745:8:4294967295] 00000400:00000200:1.0:1775776487.740571:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776487.740577:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776487.740582:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776487.740585:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776487.740589:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776487.740592:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775776487.740596:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776487.740599:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775776487.740603:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776487.740605:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776487.740608:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776487.740612:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776487.740615:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776487.740616:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776487.740619:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776487.740622:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061180 x1862022710418304/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776512 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.740631:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776487.740634:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061180 x1862022710418304/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776512 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.740641:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776487.740642:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776487.740657:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776487.740661:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776487.740665:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776487.740667:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308042764 00000100:00000001:1.0:1775776487.740669:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308046598 00000100:00000001:1.0:1775776487.740671:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308042764 00000100:00000001:1.0:1775776487.740673:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308046598 00000100:00000001:1.0:1775776487.740676:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776487.740678:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776487.740680:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776487.740684:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776487/real 1775776487] req@ffff8f622e061180 x1862022710418304/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776487.740693:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776487.740695:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776487.740697:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776487.740699:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776487.740701:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776487.740703:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776487.740706:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061180 x1862022710418304/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776487.740716:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061180 x1862022710418304/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.740722:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776487.740724:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776487.740725:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776487.740727:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776487.740731:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061180 x1862022710418304/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.740738:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776487.740742:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061180 x1862022710418304/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.740749:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776487.740751:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776487.740753:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776487.740755:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776487.740756:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776487.740758:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776487.740760:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776487.740764:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776487.740766:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776487.740771:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061180 x1862022710418304/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776487.740780:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710418304:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776487.740783:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776487.740785:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776487.740788:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061c00 x1862022710418176/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.740795:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776487.740796:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776487.740799:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776487.740801:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776487.740803:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776487.740804:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776487.740806:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776487.740807:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e6c00. 02000000:00000001:1.0:1775776487.740810:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776487.740812:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776487.740813:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061c00. 00000100:00000001:1.0:1775776487.740816:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776487.740817:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776487.740819:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776487.740822:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061180 x1862022710418304/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776512 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776487.740828:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776487.740829:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776487.740831:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776487.740833:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776487.740834:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776487.740835:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776487.740837:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776487.740839:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e5c00. 02000000:00000001:1.0:1775776487.740841:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776487.740843:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776487.740844:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061180. 00000100:00000001:1.0:1775776487.740846:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776487.740848:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776487.740850:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776487.740853:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776487.740854:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776487.740855:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776487.740857:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776487.867247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776487.867252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776487.867257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776487.867259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776487.867263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776487.867264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776487.867350:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776487.867354:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776487.867357:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1775776488.571261:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:0.0:1775776488.571284:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:0.0:1775776488.571290:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1775776488.571488:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000200:1.0:1775776488.571493:0:2405:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1775776488.571506:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775776488.571512:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776488.571516:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776488.763285:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776488.763290:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776488.763292:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776488.763297:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776488.763300:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776488.763303:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776488.763305:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776488.763307:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776488.763309:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776488.891288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776488.891294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776488.891300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776488.891302:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776488.891305:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776488.891307:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776488.891424:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776488.891430:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776488.891435:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776489.787258:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776489.787263:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776489.787265:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776489.787269:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776489.787272:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776489.787276:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776489.787278:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776489.787280:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776489.787283:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776489.915244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776489.915249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776489.915252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776489.915253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776489.915256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776489.915257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776489.915298:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776489.915303:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776489.915308:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776490.811293:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776490.811300:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776490.811301:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776490.811306:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776490.811310:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776490.811314:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776490.811316:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776490.811318:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776490.811320:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776490.939245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776490.939250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776490.939254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776490.939255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776490.939258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776490.939259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776490.939315:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776490.939319:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776490.939321:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776491.643252:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308046598, at 4308046656 00000100:00000040:0.0:1775776491.643266:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308046598, at 4308046656 00000100:00000010:1.0:1775776491.643266:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e27e0. 00000100:00000010:0.0:1775776491.643274:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2e40. 00000100:00000001:1.0:1775776491.835266:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776491.835273:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776491.835275:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776491.835281:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776491.835286:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776491.835290:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776491.835293:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776491.835295:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776491.835298:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776491.963268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776491.963276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776491.963281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776491.963284:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776491.963286:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776491.963289:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776491.963292:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776491.963294:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776491.963295:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776491.963297:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776491.963300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776491.963301:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776491.963301:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776491.963302:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776491.963303:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776491.963304:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776491.963305:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776491.963306:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776491.963393:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776491.963399:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776491.963402:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776492.859251:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776492.859252:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776492.859257:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776492.859259:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776492.859263:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1775776492.859264:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776492.859268:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776492.859268:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776492.859272:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1775776492.859273:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776492.859277:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776492.859279:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776492.859281:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776492.859285:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776492.859288:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13375 00000400:00000200:0.0:1775776492.859298:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776492.859303:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776492.859306:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776492.859308:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776492.859311:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776492.859313:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776492.859315:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776492.859318:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776492.859319:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776492.859322:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776492.859324:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776492.859326:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776492.859329:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776492.859331:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776492.859332:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776492.859334:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776492.859335:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776492.859337:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776492.859339:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776492.859341:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776492.859344:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776492.859346:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776492.859349:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776492.859350:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776492.859351:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776492.859360:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061f80. 00000020:00000040:0.0:1775776492.859363:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776492.859366:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776492.859367:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776492.859369:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776492.859371:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776492.859374:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776492.859382:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7000. 00000100:00000001:0.0:1775776492.859387:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776492.859394:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061f80 x1862022710418432/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776492.859402:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776492.859406:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776492.859409:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000029d3794] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061f80 x1862022710418432/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776492.859430:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776492.859433:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776492.859437:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776492.859439:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776492.859440:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776492.859443:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776492.859447:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13375 00000400:00000200:0.0:1775776492.859452:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776492.859457:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776492.859460:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776492.859462:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1775776492.859463:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776492.859467:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776492.859467:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776492.859470:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776492.859473:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776492.859478:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:1.0:1775776492.859479:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061f80 x1862022710418432/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775776492.859482:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776492.859484:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776492.859488:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776492.859491:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:1.0:1775776492.859494:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1775776492.859495:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776492.859497:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776492.859498:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776492.859501:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1775776492.859503:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776492.859504:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776492.859505:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776492.859509:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776492.859509:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776492.859513:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776492.859517:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00100000:1.0:1775776492.859517:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710418432:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776492.859523:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1775776492.859524:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776492.859527:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776492.859527:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776492.859528:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776492.859530:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776492.859533:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1775776492.859534:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776492.859537:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776492.859538:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000010:1.0:1775776492.859538:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c000. 02000000:00000001:1.0:1775776492.859542:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1775776492.859543:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776492.859546:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776492.859549:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776492.859550:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776492.859554:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000010:0.0:1775776492.859570:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063b80. 00000400:00000010:1.0:1775776492.859572:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000020:00000040:0.0:1775776492.859573:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776492.859577:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776492.859579:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776492.859580:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776492.859581:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:1.0:1775776492.859581:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710418432, portal 25 02000000:00000001:0.0:1775776492.859584:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776492.859585:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776492.859588:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574561152 : -123823134990464 : ffff8f622e061f80) 02000000:00000010:0.0:1775776492.859591:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7c00. 00000100:00000040:1.0:1775776492.859594:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061f80 x1862022710418432/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776517 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776492.859596:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776492.859605:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063b80 x1862022710418560/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776492.859605:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776492.859612:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:0.0:1775776492.859617:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1775776492.859617:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000001:0.0:1775776492.859621:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776492.859622:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710418432, offset 0 00000100:00000040:0.0:1775776492.859626:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ee9007b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063b80 x1862022710418560/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776492.859631:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:0.0:1775776492.859640:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776492.859640:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00080000:0.0:1775776492.859643:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776492.859647:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:1.0:1775776492.859648:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:0.0:1775776492.859649:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776492.859652:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776492.859655:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775776492.859655:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:0.0:1775776492.859660:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13375 00000400:00000200:1.0:1775776492.859660:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776492.859666:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776492.859668:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776492.859671:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453821:453821:256:4294967295] 192.168.201.115@tcp LPNI seq info [453746:453746:8:4294967295] 00000400:00000200:1.0:1775776492.859674:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776492.859682:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776492.859686:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776492.859689:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000010:1.0:1775776492.859692:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000400:00000200:0.0:1775776492.859693:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776492.859696:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776492.859699:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776492.859701:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.0:1775776492.859703:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:0.0:1775776492.859705:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776492.859708:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000200:1.0:1775776492.859709:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000020:00000001:0.0:1775776492.859710:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776492.859713:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776492.859715:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776492.859718:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000800:00000200:1.1:1775776492.859719:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:0.0:1775776492.859721:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:1.1:1775776492.859723:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000020:00000001:0.0:1775776492.859724:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775776492.859731:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776492.859734:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:1.0:1775776492.859735:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000040:0.0:1775776492.859738:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000010:1.0:1775776492.859739:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000100:00000001:0.0:1775776492.859740:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776492.859743:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776492.859745:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:0.0:1775776492.859746:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776492.859750:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776492.859752:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776492.859755:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 02000000:00000001:0.0:1775776492.859756:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776492.859758:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776492.859759:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000100:00000001:0.0:1775776492.859760:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000010:1.0:1775776492.859762:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000008:00100000:0.0:1775776492.859763:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776492.859767:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776492.859769:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776492.859770:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:0.0:1775776492.859772:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061500. 00000100:00000200:1.0:1775776492.859774:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061f80 x1862022710418432/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776517 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775776492.859775:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776492.859780:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776492.859782:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776492.859786:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776492.859789:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776492.859791:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776492.859791:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000010:0.0:1775776492.859794:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5c00. 00000100:00000040:1.0:1775776492.859795:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061f80 x1862022710418432/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776517 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776492.859799:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776492.859807:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061500 x1862022710418688/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776492.859807:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776492.859811:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776492.859817:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776492.859817:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776492.859821:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776492.859825:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:0.0:1775776492.859828:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb8d992] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061500 x1862022710418688/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.859829:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308051718 00000100:00000010:1.0:1775776492.859837:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e27e0. 00000100:00000001:0.0:1775776492.859838:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776492.859841:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776492.859844:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776492.859846:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776492.859846:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000040:0.0:1775776492.859851:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776492.859855:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13385) 00000100:00000001:1.0:1775776492.859855:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308051718 00000100:00000010:1.0:1775776492.859859:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2ba0. 00000100:00000001:1.0:1775776492.859862:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776492.859865:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776492.859868:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776492.859870:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776492.859874:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776492/real 1775776492] req@ffff8f622e061f80 x1862022710418432/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776492.859881:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776492.859883:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776492.859884:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776492.859886:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776492.859887:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776492.859889:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776492.859891:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061f80 x1862022710418432/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776492.859897:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061f80 x1862022710418432/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.859901:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776492.859902:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776492.859904:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776492.859906:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776492.859908:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061f80 x1862022710418432/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.859912:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776492.859915:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061f80 x1862022710418432/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.859919:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776492.859921:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776492.859922:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776492.859923:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776492.859924:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776492.859925:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776492.859928:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776492.859930:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776492.859932:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776492.859935:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061f80 x1862022710418432/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776492.859940:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710418432:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776492.859943:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776492.859945:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776492.859947:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061f80 x1862022710418432/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.859951:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776492.859953:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776492.859955:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c000. 02000000:00000001:1.0:1775776492.859956:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776492.859958:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776492.859959:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776492.859960:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776492.859962:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e7000. 02000000:00000001:1.0:1775776492.859964:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776492.859965:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776492.859967:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061f80. 00000100:00000001:1.0:1775776492.859969:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776492.859970:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776492.859971:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776492.859976:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776492.859977:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776492.859979:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776492.859980:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776492.859981:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776492.859983:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e063b80 x1862022710418560/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.859988:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776492.859989:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776492.859990:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776492.859992:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776492.859993:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776492.859995:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e063b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710418560:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776492.859997:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776492.859998:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776492.859999:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776492.860001:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776492.860003:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c000. 02000000:00000001:1.0:1775776492.860004:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776492.860007:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776492.860008:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776492.860010:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776492.860012:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710418560, portal 10 00000100:00000001:1.0:1775776492.860014:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776492.860015:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574568320 : -123823134983296 : ffff8f622e063b80) 00000100:00000040:1.0:1775776492.860019:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e063b80 x1862022710418560/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776517 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.860023:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776492.860025:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776492.860026:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776492.860028:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710418560, offset 0 00000400:00000200:1.0:1775776492.860030:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776492.860033:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776492.860035:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776492.860037:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776492.860039:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776492.860040:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776492.860042:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776492.860044:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453822:453822:256:4294967295] 192.168.201.115@tcp LPNI seq info [453747:453747:8:4294967295] 00000400:00000200:1.0:1775776492.860047:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776492.860051:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776492.860054:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776492.860056:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776492.860059:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776492.860061:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775776492.860064:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776492.860065:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775776492.860068:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776492.860070:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776492.860072:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776492.860074:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776492.860076:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776492.860077:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776492.860079:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776492.860081:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e063b80 x1862022710418560/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776517 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.860086:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776492.860088:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e063b80 x1862022710418560/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776517 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.860092:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776492.860093:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776492.860094:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776492.860097:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776492.860099:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776492.860100:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308047883 00000100:00000001:1.0:1775776492.860102:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308051718 00000100:00000001:1.0:1775776492.860103:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308047883 00000100:00000001:1.0:1775776492.860105:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308051718 00000100:00000001:1.0:1775776492.860106:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776492.860108:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776492.860109:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776492.860111:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776492/real 1775776492] req@ffff8f622e063b80 x1862022710418560/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776492.860115:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776492.860116:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776492.860117:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776492.860118:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776492.860119:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776492.860121:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776492.860123:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e063b80 x1862022710418560/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776492.860128:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e063b80 x1862022710418560/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.860149:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776492.860150:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776492.860151:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776492.860152:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776492.860155:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e063b80 x1862022710418560/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.860159:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776492.860162:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e063b80 x1862022710418560/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.860166:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776492.860167:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776492.860168:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776492.860169:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776492.860170:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776492.860174:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776492.860175:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776492.860177:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776492.860178:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776492.860181:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e063b80 x1862022710418560/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776492.860185:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e063b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710418560:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776492.860188:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776492.860190:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061500 x1862022710418688/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.860199:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776492.860200:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776492.860202:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776492.860203:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776492.860204:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776492.860206:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710418688:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776492.860208:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776492.860209:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776492.860210:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776492.860211:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776492.860213:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776492.860214:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776492.860216:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776492.860217:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776492.860219:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776492.860221:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710418688, portal 10 00000100:00000001:1.0:1775776492.860222:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776492.860223:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574558464 : -123823134993152 : ffff8f622e061500) 00000100:00000040:1.0:1775776492.860227:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061500 x1862022710418688/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776517 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.860231:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776492.860233:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776492.860234:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776492.860235:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710418688, offset 0 00000400:00000200:1.0:1775776492.860238:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776492.860240:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776492.860243:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776492.860244:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776492.860246:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776492.860248:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776492.860249:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776492.860252:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453823:453823:256:4294967295] 192.168.201.115@tcp LPNI seq info [453748:453748:8:4294967295] 00000400:00000200:1.0:1775776492.860255:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776492.860259:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776492.860262:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776492.860264:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776492.860267:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776492.860269:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775776492.860271:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776492.860273:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775776492.860275:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776492.860276:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776492.860278:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776492.860280:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776492.860282:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776492.860283:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776492.860285:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776492.860287:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061500 x1862022710418688/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776517 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.860292:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776492.860294:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061500 x1862022710418688/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776517 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.860298:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776492.860299:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776492.860300:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776492.860302:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776492.860303:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776492.860305:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308047884 00000100:00000001:1.0:1775776492.860306:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308051718 00000100:00000001:1.0:1775776492.860307:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308047884 00000100:00000001:1.0:1775776492.860309:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308051718 00000100:00000001:1.0:1775776492.860310:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776492.860312:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776492.860313:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776492.860316:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776492/real 1775776492] req@ffff8f622e061500 x1862022710418688/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776492.860321:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776492.860322:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776492.860323:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776492.860324:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776492.860325:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776492.860327:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776492.860328:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061500 x1862022710418688/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776492.860334:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061500 x1862022710418688/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.860338:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776492.860339:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776492.860340:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776492.860341:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776492.860343:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061500 x1862022710418688/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.860347:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776492.860349:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061500 x1862022710418688/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.860353:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776492.860354:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776492.860355:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776492.860356:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776492.860357:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776492.860358:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776492.860360:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776492.860363:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776492.860365:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776492.860369:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061500 x1862022710418688/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776492.860377:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710418688:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776492.860381:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776492.860383:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776492.860386:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e063b80 x1862022710418560/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.860393:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776492.860394:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776492.860396:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c000. 02000000:00000001:1.0:1775776492.860399:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776492.860400:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776492.860402:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776492.860404:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776492.860406:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e7c00. 02000000:00000001:1.0:1775776492.860408:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776492.860410:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776492.860411:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e063b80. 00000100:00000001:1.0:1775776492.860414:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776492.860415:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776492.860417:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776492.860421:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061500 x1862022710418688/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776517 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776492.860428:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776492.860429:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776492.860431:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776492.860433:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776492.860434:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776492.860435:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776492.860436:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776492.860438:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e5c00. 02000000:00000001:1.0:1775776492.860439:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776492.860440:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776492.860441:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061500. 00000100:00000001:1.0:1775776492.860442:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776492.860443:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776492.860445:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776492.860447:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776492.860448:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776492.860449:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776492.860450:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776492.987281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776492.987286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776492.987290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776492.987292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776492.987295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776492.987296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776492.987395:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776492.987401:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776492.987406:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776493.883341:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776493.883349:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776493.883351:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776493.883358:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776493.883362:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776493.883367:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776493.883370:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776493.883372:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776493.883375:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776494.011255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776494.011261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776494.011266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776494.011267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776494.011270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776494.011271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776494.011340:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776494.011344:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776494.011348:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776494.907254:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776494.907259:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776494.907260:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776494.907265:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776494.907267:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776494.907270:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776494.907272:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776494.907274:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776494.907276:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776495.035249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776495.035255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776495.035260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776495.035261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776495.035264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776495.035265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776495.035330:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776495.035334:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776495.035337:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776495.931245:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776495.931252:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776495.931254:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776495.931259:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776495.931262:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776495.931266:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776495.931268:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776495.931270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776495.931272:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776496.059244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776496.059249:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776496.059251:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776496.059254:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776496.059258:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776496.059260:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776496.059262:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776496.059264:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776496.059268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776496.059269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776496.059272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776496.059279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776496.059335:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776496.059338:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776496.059342:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776496.763239:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308051718, at 4308051776 00000100:00000040:1.0:1775776496.763239:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308051718, at 4308051776 00000100:00000010:0.0:1775776496.763254:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e27e0. 00000100:00000010:1.0:1775776496.763256:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ba0. 00000100:00000001:1.0:1775776496.955223:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776496.955228:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776496.955229:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776496.955233:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776496.955236:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776496.955238:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776496.955240:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776496.955242:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776496.955244:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776497.083240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776497.083244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776497.083247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776497.083248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776497.083251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776497.083251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776497.083301:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776497.083305:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776497.083307:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776497.979228:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776497.979234:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776497.979236:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776497.979239:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775776497.979240:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776497.979245:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776497.979246:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776497.979248:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776497.979248:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776497.979252:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775776497.979252:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776497.979254:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776497.979258:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776497.979259:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776497.979265:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13380 00000400:00000200:3.0:1775776497.979272:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776497.979276:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776497.979278:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776497.979280:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776497.979282:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776497.979285:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776497.979286:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776497.979289:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776497.979290:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776497.979292:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776497.979294:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776497.979295:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776497.979297:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776497.979298:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776497.979299:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776497.979300:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776497.979301:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776497.979303:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776497.979305:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776497.979306:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776497.979308:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776497.979310:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776497.979311:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776497.979313:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776497.979314:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776497.979321:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775776497.979324:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776497.979328:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776497.979329:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776497.979332:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776497.979334:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776497.979338:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776497.979345:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000100:00000001:3.0:1775776497.979351:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776497.979359:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710418816/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776497.979369:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776497.979378:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776497.979382:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710418816/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776497.979395:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776497.979396:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776497.979398:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776497.979399:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776497.979401:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776497.979402:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776497.979404:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13380 00000400:00000200:3.0:1775776497.979407:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776497.979410:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776497.979412:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776497.979414:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1775776497.979414:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776497.979417:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776497.979417:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776497.979419:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776497.979421:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776497.979422:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776497.979424:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776497.979426:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000040:1.0:1775776497.979427:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71c00 x1862022710418816/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775776497.979427:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776497.979430:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776497.979434:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776497.979437:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776497.979442:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1775776497.979443:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776497.979444:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776497.979446:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775776497.979449:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1775776497.979449:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776497.979452:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776497.979453:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1775776497.979453:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776497.979458:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000001:1.0:1775776497.979459:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776497.979460:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776497.979463:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1775776497.979475:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710418816:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776497.979480:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776497.979480:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776497.979483:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776497.979484:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1775776497.979484:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776497.979488:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775776497.979489:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775776497.979491:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:3.0:1775776497.979492:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776497.979494:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776497.979497:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 00000100:00000010:3.0:1775776497.979497:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 02000000:00000001:1.0:1775776497.979502:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776497.979502:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776497.979507:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776497.979508:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776497.979509:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776497.979510:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776497.979512:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776497.979513:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000010:3.0:1775776497.979516:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000400:00000010:1.0:1775776497.979519:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000001:3.0:1775776497.979519:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776497.979526:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710418944/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1775776497.979528:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000001:3.0:1775776497.979533:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776497.979535:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776497.979537:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710418816, portal 25 00000100:00000040:3.0:1775776497.979538:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710418944/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.979541:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776497.979544:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849011200 : -123823860540416 : ffff8f6202c71c00) 00000100:00000001:3.0:1775776497.979550:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776497.979552:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1775776497.979555:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71c00 x1862022710418816/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776522 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775776497.979556:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776497.979558:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776497.979560:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776497.979565:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776497.979569:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1775776497.979569:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13380 00000100:00000040:1.0:1775776497.979574:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000200:3.0:1775776497.979576:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776497.979578:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000400:00000200:3.0:1775776497.979580:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000200:1.0:1775776497.979581:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710418816, offset 0 00000100:00080000:3.0:1775776497.979582:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776497.979586:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776497.979588:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776497.979591:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776497.979592:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776497.979595:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776497.979596:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776497.979597:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776497.979598:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:1.0:1775776497.979600:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000040:3.0:1775776497.979600:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776497.979603:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776497.979605:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776497.979607:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000040:3.0:1775776497.979610:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776497.979612:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776497.979613:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776497.979617:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000040:3.0:1775776497.979619:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776497.979621:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776497.979622:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000020:00000040:3.0:1775776497.979624:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776497.979626:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00080000:3.0:1775776497.979630:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000400:00000200:1.0:1775776497.979633:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453824:453824:256:4294967295] 192.168.201.115@tcp LPNI seq info [453749:453749:8:4294967295] 00000100:00000001:3.0:1775776497.979636:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776497.979640:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776497.979641:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776497.979645:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775776497.979651:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 02000000:00000001:3.0:1775776497.979651:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776497.979656:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:1.0:1775776497.979659:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000008:00100000:3.0:1775776497.979659:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000010:1.0:1775776497.979663:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000008:00000001:3.0:1775776497.979663:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776497.979668:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776497.979671:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000010:3.0:1775776497.979672:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000800:00000200:1.0:1775776497.979675:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000020:00000040:3.0:1775776497.979676:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776497.979682:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000800:00000200:1.1:1775776497.979683:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:3.0:1775776497.979685:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776497.979687:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000100:00000001:3.0:1775776497.979688:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000100:1.0:1775776497.979691:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 02000000:00000001:3.0:1775776497.979691:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000010:1.0:1775776497.979694:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 02000000:00000001:3.0:1775776497.979694:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776497.979700:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000400:00000200:1.0:1775776497.979701:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1775776497.979703:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776497.979706:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00100000:3.0:1775776497.979710:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710419072/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776497.979711:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776497.979714:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776497.979720:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1775776497.979720:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776497.979722:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776497.979726:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71c00 x1862022710418816/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776522 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776497.979727:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710419072/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.979745:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1775776497.979746:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776497.979749:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1775776497.979751:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71c00 x1862022710418816/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776522 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776497.979752:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776497.979757:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776497.979763:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776497.979767:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776497.979769:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1775776497.979769:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13390) 00000100:00000001:1.0:1775776497.979775:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776497.979778:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776497.979780:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308056838 00000100:00000010:1.0:1775776497.979783:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2ba0. 00000100:00000001:1.0:1775776497.979788:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776497.979792:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308056838 00000100:00000010:1.0:1775776497.979794:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e20c0. 00000100:00000001:1.0:1775776497.979796:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776497.979798:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776497.979800:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776497.979801:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776497.979804:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776497/real 1775776497] req@ffff8f6202c71c00 x1862022710418816/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776497.979809:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776497.979811:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776497.979812:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776497.979813:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776497.979814:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776497.979816:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776497.979818:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71c00 x1862022710418816/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776497.979823:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71c00 x1862022710418816/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.979826:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776497.979827:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776497.979829:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776497.979830:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776497.979832:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71c00 x1862022710418816/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.979836:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776497.979838:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71c00 x1862022710418816/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.979842:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776497.979844:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776497.979845:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776497.979845:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776497.979847:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776497.979848:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776497.979850:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776497.979852:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776497.979854:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776497.979856:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71c00 x1862022710418816/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776497.979860:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710418816:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776497.979863:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776497.979864:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776497.979866:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71c00 x1862022710418816/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.979870:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776497.979871:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776497.979873:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776497.979874:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776497.979875:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776497.979876:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776497.979877:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776497.979878:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36c00. 02000000:00000001:1.0:1775776497.979881:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776497.979882:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776497.979883:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71c00. 00000100:00000001:1.0:1775776497.979885:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776497.979886:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776497.979887:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776497.979892:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776497.979893:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776497.979894:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776497.979895:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776497.979896:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776497.979897:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70e00 x1862022710418944/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.979902:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776497.979902:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776497.979904:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776497.979905:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776497.979906:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776497.979907:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710418944:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776497.979909:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776497.979910:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776497.979911:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776497.979912:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776497.979914:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776497.979915:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776497.979917:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776497.979919:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776497.979920:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776497.979922:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710418944, portal 10 00000100:00000001:1.0:1775776497.979923:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776497.979924:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849007616 : -123823860544000 : ffff8f6202c70e00) 00000100:00000040:1.0:1775776497.979929:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70e00 x1862022710418944/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776522 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.979935:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776497.979937:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776497.979939:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776497.979941:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710418944, offset 0 00000400:00000200:1.0:1775776497.979954:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776497.979958:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776497.979961:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776497.979964:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776497.979966:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776497.979968:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776497.979970:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776497.979973:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453825:453825:256:4294967295] 192.168.201.115@tcp LPNI seq info [453750:453750:8:4294967295] 00000400:00000200:1.0:1775776497.979977:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776497.979982:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776497.979986:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776497.979990:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776497.979993:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776497.979996:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775776497.980000:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776497.980002:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1775776497.980005:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776497.980007:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776497.980010:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776497.980013:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776497.980015:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776497.980017:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776497.980019:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776497.980022:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70e00 x1862022710418944/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776522 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.980029:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776497.980032:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70e00 x1862022710418944/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776522 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.980048:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776497.980049:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776497.980052:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776497.980056:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776497.980059:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776497.980060:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308053003 00000100:00000001:1.0:1775776497.980063:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308056838 00000100:00000001:1.0:1775776497.980064:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308053003 00000100:00000001:1.0:1775776497.980066:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308056838 00000100:00000001:1.0:1775776497.980067:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776497.980068:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776497.980069:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776497.980072:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776497/real 1775776497] req@ffff8f6202c70e00 x1862022710418944/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776497.980076:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776497.980077:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776497.980078:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776497.980080:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776497.980081:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776497.980082:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776497.980084:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70e00 x1862022710418944/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776497.980089:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70e00 x1862022710418944/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.980092:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776497.980093:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776497.980094:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776497.980095:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776497.980096:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70e00 x1862022710418944/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.980100:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776497.980102:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70e00 x1862022710418944/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.980106:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776497.980107:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776497.980108:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776497.980109:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776497.980110:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776497.980113:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776497.980114:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776497.980116:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776497.980117:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776497.980119:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70e00 x1862022710418944/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776497.980123:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710418944:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776497.980126:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776497.980127:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70000 x1862022710419072/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.980151:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776497.980152:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776497.980154:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776497.980155:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776497.980156:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776497.980157:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710419072:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776497.980159:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776497.980160:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776497.980161:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776497.980162:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776497.980164:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c000. 02000000:00000001:1.0:1775776497.980165:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776497.980168:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776497.980169:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776497.980171:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776497.980173:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710419072, portal 10 00000100:00000001:1.0:1775776497.980174:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776497.980175:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004032 : -123823860547584 : ffff8f6202c70000) 00000100:00000040:1.0:1775776497.980179:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70000 x1862022710419072/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776522 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.980183:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776497.980184:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776497.980185:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776497.980186:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710419072, offset 0 00000400:00000200:1.0:1775776497.980189:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776497.980191:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776497.980193:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776497.980195:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776497.980197:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776497.980200:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776497.980202:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776497.980205:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453826:453826:256:4294967295] 192.168.201.115@tcp LPNI seq info [453751:453751:8:4294967295] 00000400:00000200:1.0:1775776497.980209:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776497.980214:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776497.980218:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776497.980221:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776497.980224:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776497.980226:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775776497.980230:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776497.980232:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1775776497.980236:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776497.980237:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776497.980240:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776497.980243:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776497.980245:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776497.980247:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776497.980249:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776497.980253:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70000 x1862022710419072/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776522 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.980260:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776497.980262:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70000 x1862022710419072/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776522 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.980268:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776497.980269:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776497.980272:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776497.980275:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776497.980278:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776497.980280:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308053004 00000100:00000001:1.0:1775776497.980282:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308056838 00000100:00000001:1.0:1775776497.980284:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308053004 00000100:00000001:1.0:1775776497.980286:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308056838 00000100:00000001:1.0:1775776497.980289:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776497.980291:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776497.980293:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776497.980296:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776497/real 1775776497] req@ffff8f6202c70000 x1862022710419072/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776497.980303:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776497.980305:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776497.980306:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776497.980308:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776497.980310:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776497.980312:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776497.980314:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70000 x1862022710419072/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776497.980322:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70000 x1862022710419072/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.980326:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776497.980326:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776497.980327:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776497.980328:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776497.980330:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70000 x1862022710419072/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.980334:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776497.980336:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70000 x1862022710419072/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.980340:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776497.980341:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776497.980342:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776497.980343:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776497.980344:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776497.980345:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776497.980346:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776497.980348:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776497.980349:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776497.980352:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70000 x1862022710419072/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776497.980356:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710419072:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776497.980358:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776497.980359:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776497.980360:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70e00 x1862022710418944/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.980364:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776497.980364:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776497.980365:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776497.980367:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776497.980368:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776497.980368:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776497.980370:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776497.980371:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34000. 02000000:00000001:1.0:1775776497.980372:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776497.980373:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776497.980374:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70e00. 00000100:00000001:1.0:1775776497.980375:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776497.980376:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776497.980377:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776497.980379:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70000 x1862022710419072/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776522 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776497.980382:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776497.980382:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776497.980383:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c000. 02000000:00000001:1.0:1775776497.980384:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776497.980385:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776497.980386:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776497.980387:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776497.980388:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35c00. 02000000:00000001:1.0:1775776497.980390:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776497.980390:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776497.980391:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70000. 00000100:00000001:1.0:1775776497.980392:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776497.980393:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776497.980394:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776497.980396:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776497.980397:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776497.980398:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776497.980399:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776498.107232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776498.107236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776498.107240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776498.107241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776498.107243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776498.107244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776498.107298:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776498.107302:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776498.107304:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776499.003229:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776499.003235:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776499.003237:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776499.003242:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776499.003245:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776499.003248:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776499.003250:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776499.003253:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776499.003255:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776499.131247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776499.131253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776499.131257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776499.131258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776499.131261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776499.131262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776499.131338:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776499.131342:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776499.131346:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776500.027254:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776500.027261:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776500.027263:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776500.027269:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776500.027273:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776500.027277:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776500.027280:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776500.027282:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776500.027285:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776500.155250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776500.155254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776500.155258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776500.155259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776500.155262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776500.155263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776500.155328:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776500.155332:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776500.155336:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776501.051215:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776501.051222:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776501.051224:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776501.051230:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776501.051234:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776501.051237:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776501.051247:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776501.051251:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776501.051254:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776501.179280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776501.179286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776501.179292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776501.179293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776501.179297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776501.179299:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776501.179388:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776501.179393:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776501.179396:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776501.883242:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308056838, at 4308056896 00000100:00000010:1.0:1775776501.883255:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e20c0. 00000100:00000040:0.0:1775776501.883271:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308056838, at 4308056896 00000100:00000010:0.0:1775776501.883279:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ba0. 00000100:00000001:1.0:1775776502.075268:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776502.075276:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776502.075278:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776502.075284:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776502.075288:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776502.075292:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776502.075295:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776502.075297:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776502.075300:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776502.203211:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776502.203217:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776502.203223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776502.203227:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776502.203229:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776502.203233:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776502.203235:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776502.203238:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776502.203240:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776502.203242:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776502.203247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776502.203248:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776502.203249:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776502.203251:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776502.203253:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776502.203254:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776502.203256:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776502.203258:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776502.203316:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776502.203321:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776502.203324:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775776503.099242:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776503.099253:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776503.099256:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776503.099260:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776503.099265:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776503.099273:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13385 00000400:00000200:1.0:1775776503.099283:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776503.099289:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776503.099293:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776503.099296:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776503.099298:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776503.099300:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776503.099302:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:1.0:1775776503.099305:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776503.099306:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:1.0:1775776503.099309:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776503.099312:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:1.0:1775776503.099314:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:1.0:1775776503.099315:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:1.0:1775776503.099317:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776503.099318:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776503.099320:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776503.099321:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:1.0:1775776503.099323:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:1.0:1775776503.099325:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776503.099326:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776503.099329:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776503.099331:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776503.099333:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776503.099335:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776503.099336:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776503.099345:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6234919f80. 00000020:00000040:1.0:1775776503.099347:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:1.0:1775776503.099350:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776503.099351:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776503.099354:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776503.099356:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776503.099359:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776503.099366:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4c000. 00000100:00000001:1.0:1775776503.099371:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776503.099401:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6234919f80 x1862022710419200/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776503.099410:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776503.099414:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776503.099417:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0867d60] to pc [ptlrpcd_rcv+-1] req@ffff8f6234919f80 x1862022710419200/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776503.099436:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776503.099439:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776503.099443:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776503.099444:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776503.099445:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776503.099447:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776503.099449:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13385 00000400:00000200:1.0:1775776503.099452:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776503.099454:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776503.099455:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776503.099457:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776503.099459:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776503.099460:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776503.099462:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:1.0:1775776503.099463:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776503.099465:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:1.0:1775776503.099466:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776503.099467:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:1.0:1775776503.099469:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:1.0:1775776503.099470:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:1.0:1775776503.099472:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776503.099474:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776503.099475:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776503.099476:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:1.0:1775776503.099478:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:1.0:1775776503.099480:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776503.099482:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776503.099483:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775776503.099486:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1775776503.099488:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776503.099491:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1775776503.099491:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776503.099493:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776503.099495:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000008:00100000:1.0:1775776503.099497:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776503.099501:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776503.099504:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776503.099506:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234919f80 x1862022710419200/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1775776503.099507:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623491a300. 00000020:00000040:1.0:1775776503.099511:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776503.099515:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776503.099517:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776503.099521:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776503.099523:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776503.099526:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776503.099528:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000010:1.0:1775776503.099530:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4d800. 00000100:00000001:0.0:1775776503.099531:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776503.099536:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776503.099538:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776503.099541:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1775776503.099543:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623491a300 x1862022710419328/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776503.099550:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776503.099553:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776503.099554:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776503.099557:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bfa067b] to pc [ptlrpcd_rcv+-1] req@ffff8f623491a300 x1862022710419328/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776503.099562:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234919f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710419200:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776503.099569:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776503.099573:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776503.099580:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:1.0:1775776503.099580:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776503.099581:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776503.099584:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1775776503.099586:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1775776503.099586:0:191261:0:(import.c:484:import_select_connection()) Process entered 02000000:00000001:0.0:1775776503.099590:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776503.099591:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13385 02000000:00000001:0.0:1775776503.099596:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1775776503.099596:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776503.099600:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776503.099602:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 02000000:00000010:0.0:1775776503.099604:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 00000100:00000040:1.0:1775776503.099604:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776503.099607:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775776503.099608:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776503.099610:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776503.099613:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:1.0:1775776503.099615:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776503.099617:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000400:00000200:0.0:1775776503.099618:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000001:1.0:1775776503.099620:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000010:0.0:1775776503.099623:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000020:00000040:1.0:1775776503.099623:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:1.0:1775776503.099628:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000010:0.0:1775776503.099630:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000020:00000001:1.0:1775776503.099631:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:1.0:1775776503.099637:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000200:0.0:1775776503.099638:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710419200, portal 25 00000100:00000001:1.0:1775776503.099641:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776503.099642:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776503.099645:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684366720 : -123823025184896 : ffff8f6234919f80) 00000100:00000040:1.0:1775776503.099648:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776503.099652:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:0.0:1775776503.099653:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234919f80 x1862022710419200/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776528 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1775776503.099657:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00000001:0.0:1775776503.099664:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1775776503.099666:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000040:0.0:1775776503.099671:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:1.0:1775776503.099672:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776503.099673:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 02000000:00000001:1.0:1775776503.099676:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:0.0:1775776503.099677:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710419200, offset 0 02000000:00000001:1.0:1775776503.099681:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1775776503.099686:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 02000000:00000001:1.0:1775776503.099686:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776503.099690:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:1.0:1775776503.099693:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776503.099695:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776503.099697:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776503.099698:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000010:1.0:1775776503.099703:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6234919c00. 00000400:00000200:0.0:1775776503.099706:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000020:00000040:1.0:1775776503.099707:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:0.0:1775776503.099712:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776503.099715:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 02000000:00000001:1.0:1775776503.099718:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1775776503.099720:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 02000000:00000001:1.0:1775776503.099721:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776503.099724:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:1.0:1775776503.099724:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776503.099727:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776503.099729:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776503.099732:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453827:453827:256:4294967295] 192.168.201.115@tcp LPNI seq info [453752:453752:8:4294967295] 02000000:00000010:1.0:1775776503.099735:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4f400. 00000400:00000200:0.0:1775776503.099739:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776503.099741:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776503.099747:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6234919c00 x1862022710419456/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775776503.099754:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776503.099769:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776503.099775:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000100:00000001:1.0:1775776503.099781:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:0.0:1775776503.099784:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:1.0:1775776503.099785:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775776503.099790:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000100:00000040:1.0:1775776503.099795:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000718661bc] to pc [ptlrpcd_rcv+-1] req@ffff8f6234919c00 x1862022710419456/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1775776503.099796:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776503.099803:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000100:00000001:1.0:1775776503.099813:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1775776503.099814:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000040:1.0:1775776503.099819:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:0.0:1775776503.099820:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000100:00000040:1.0:1775776503.099827:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:0.0:1775776503.099829:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:1.0:1775776503.099831:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1775776503.099836:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:0.0:1775776503.099841:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000040:1.0:1775776503.099845:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13395) 00000400:00000200:0.0:1775776503.099848:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776503.099852:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776503.099857:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776503.099862:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234919f80 x1862022710419200/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776528 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776503.099871:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776503.099873:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1775776503.099875:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1775776503.099879:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234919f80 x1862022710419200/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776528 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776503.099879:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776503.099884:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776503.099889:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776503.099894:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776503.099895:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776503.099897:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776503.099899:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776503.099900:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776503.099905:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776503.099907:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776503.099913:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776503.099917:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308061958 00000100:00000010:0.0:1775776503.099923:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3a20. 00000100:00000001:0.0:1775776503.099930:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776503.099937:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308061958 00000100:00000010:0.0:1775776503.099940:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c30c0. 00000100:00000001:0.0:1775776503.099943:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776503.099947:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776503.099949:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776503.099952:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776503.099957:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776503/real 1775776503] req@ffff8f6234919f80 x1862022710419200/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776503.099968:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776503.099969:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776503.099972:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776503.099975:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776503.099977:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776503.099980:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776503.099984:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234919f80 x1862022710419200/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776503.099993:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234919f80 x1862022710419200/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776503.100000:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776503.100002:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776503.100004:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776503.100007:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776503.100011:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234919f80 x1862022710419200/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776503.100018:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776503.100023:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234919f80 x1862022710419200/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776503.100030:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776503.100033:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776503.100034:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776503.100036:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776503.100038:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776503.100040:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776503.100043:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776503.100047:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776503.100050:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776503.100054:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234919f80 x1862022710419200/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776503.100063:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234919f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710419200:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776503.100070:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776503.100072:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776503.100076:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234919f80 x1862022710419200/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776503.100083:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776503.100086:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776503.100089:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776503.100092:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776503.100094:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776503.100096:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776503.100098:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776503.100101:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4c000. 02000000:00000001:0.0:1775776503.100104:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776503.100106:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776503.100108:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234919f80. 00000100:00000001:0.0:1775776503.100111:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776503.100113:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776503.100115:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776503.100122:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776503.100123:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776503.100126:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776503.100127:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776503.100129:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776503.100157:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491a300 x1862022710419328/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776503.100165:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776503.100167:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776503.100169:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776503.100171:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776503.100173:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776503.100177:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710419328:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776503.100180:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776503.100182:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776503.100184:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776503.100186:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776503.100190:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776503.100192:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776503.100196:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776503.100199:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776503.100202:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776503.100206:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710419328, portal 10 00000100:00000001:0.0:1775776503.100208:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776503.100209:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684367616 : -123823025184000 : ffff8f623491a300) 00000100:00000040:0.0:1775776503.100217:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491a300 x1862022710419328/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776528 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776503.100223:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776503.100226:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776503.100228:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776503.100231:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710419328, offset 0 00000400:00000200:0.0:1775776503.100235:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776503.100240:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776503.100244:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776503.100247:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776503.100250:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776503.100253:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776503.100255:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776503.100258:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453828:453828:256:4294967295] 192.168.201.115@tcp LPNI seq info [453753:453753:8:4294967295] 00000400:00000200:0.0:1775776503.100262:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776503.100269:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776503.100274:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776503.100277:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776503.100282:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776503.100285:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775776503.100289:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776503.100292:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775776503.100296:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776503.100299:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776503.100302:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776503.100306:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776503.100309:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776503.100311:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776503.100313:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776503.100317:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491a300 x1862022710419328/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776528 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776503.100325:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776503.100328:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491a300 x1862022710419328/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776528 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776503.100334:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776503.100335:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776503.100339:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776503.100343:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776503.100346:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776503.100348:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308058124 00000100:00000001:0.0:1775776503.100351:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308061958 00000100:00000001:0.0:1775776503.100354:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308058124 00000100:00000001:0.0:1775776503.100356:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308061958 00000100:00000001:0.0:1775776503.100358:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776503.100360:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776503.100362:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776503.100366:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776503/real 1775776503] req@ffff8f623491a300 x1862022710419328/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776503.100375:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776503.100429:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776503.100432:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776503.100435:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776503.100437:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776503.100440:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776503.100445:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491a300 x1862022710419328/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776503.100454:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491a300 x1862022710419328/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776503.100461:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776503.100463:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776503.100465:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776503.100466:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776503.100470:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491a300 x1862022710419328/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776503.100477:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776503.100481:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491a300 x1862022710419328/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776503.100488:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776503.100491:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776503.100492:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776503.100494:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776503.100496:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776503.100501:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776503.100503:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776503.100507:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776503.100510:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776503.100514:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491a300 x1862022710419328/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776503.100523:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710419328:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776503.100527:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776503.100530:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234919c00 x1862022710419456/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776503.100544:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776503.100545:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776503.100548:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776503.100550:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776503.100552:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776503.100555:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234919c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710419456:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776503.100558:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776503.100561:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776503.100562:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776503.100564:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776503.100568:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775776503.100570:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776503.100574:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776503.100577:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776503.100580:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776503.100583:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710419456, portal 10 00000100:00000001:0.0:1775776503.100585:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776503.100586:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684365824 : -123823025185792 : ffff8f6234919c00) 00000100:00000040:0.0:1775776503.100592:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234919c00 x1862022710419456/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776528 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776503.100599:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776503.100601:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776503.100603:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776503.100606:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710419456, offset 0 00000400:00000200:0.0:1775776503.100610:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776503.100614:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776503.100618:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776503.100621:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776503.100623:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776503.100626:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776503.100628:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776503.100632:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453829:453829:256:4294967295] 192.168.201.115@tcp LPNI seq info [453754:453754:8:4294967295] 00000400:00000200:0.0:1775776503.100637:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776503.100644:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776503.100649:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776503.100653:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776503.100657:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776503.100660:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775776503.100665:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776503.100668:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775776503.100672:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776503.100674:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776503.100677:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776503.100681:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776503.100684:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776503.100686:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776503.100688:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776503.100692:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234919c00 x1862022710419456/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776528 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776503.100701:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776503.100704:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234919c00 x1862022710419456/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776528 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776503.100712:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776503.100714:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776503.100717:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776503.100720:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776503.100724:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776503.100726:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308058124 00000100:00000001:0.0:1775776503.100728:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308061958 00000100:00000001:0.0:1775776503.100731:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308058124 00000100:00000001:0.0:1775776503.100733:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308061958 00000100:00000001:0.0:1775776503.100736:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776503.100738:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776503.100740:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776503.100744:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776503/real 1775776503] req@ffff8f6234919c00 x1862022710419456/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776503.100753:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776503.100754:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776503.100756:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776503.100759:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776503.100760:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776503.100761:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776503.100764:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234919c00 x1862022710419456/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776503.100770:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234919c00 x1862022710419456/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776503.100774:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776503.100775:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776503.100776:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776503.100776:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776503.100779:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234919c00 x1862022710419456/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776503.100783:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776503.100786:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234919c00 x1862022710419456/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776503.100790:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776503.100792:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776503.100793:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776503.100794:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776503.100795:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776503.100796:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776503.100797:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776503.100800:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776503.100801:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776503.100804:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234919c00 x1862022710419456/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776503.100809:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234919c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710419456:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776503.100811:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776503.100812:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776503.100814:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491a300 x1862022710419328/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776503.100818:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776503.100819:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776503.100820:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776503.100822:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776503.100823:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776503.100824:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776503.100825:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776503.100826:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:0.0:1775776503.100828:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776503.100829:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776503.100830:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491a300. 00000100:00000001:0.0:1775776503.100832:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776503.100834:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776503.100835:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776503.100838:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234919c00 x1862022710419456/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776528 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776503.100845:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776503.100846:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776503.100848:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775776503.100850:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776503.100851:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776503.100853:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776503.100855:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776503.100857:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4f400. 02000000:00000001:0.0:1775776503.100859:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776503.100861:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776503.100862:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234919c00. 00000100:00000001:0.0:1775776503.100864:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776503.100865:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776503.100867:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776503.100870:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776503.100872:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776503.100874:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776503.100876:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776503.227257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776503.227263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776503.227268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776503.227270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776503.227274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776503.227275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776503.227330:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776503.227333:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776503.227336:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776504.123200:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776504.123206:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776504.123208:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776504.123212:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776504.123214:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776504.123217:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776504.123219:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776504.123221:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776504.123222:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776504.251244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776504.251249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776504.251253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776504.251254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776504.251257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776504.251258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776504.251312:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776504.251315:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776504.251318:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776505.147267:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776505.147275:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776505.147277:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776505.147283:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776505.147288:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776505.147291:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776505.147294:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776505.147297:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776505.147299:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776505.275267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776505.275272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776505.275276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776505.275277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776505.275280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776505.275281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776505.275378:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776505.275383:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776505.275386:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776506.171227:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776506.171235:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776506.171237:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776506.171244:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776506.171248:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776506.171252:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776506.171255:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776506.171257:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776506.171260:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776506.299261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776506.299268:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776506.299270:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776506.299272:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776506.299276:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776506.299278:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776506.299280:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776506.299281:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776506.299284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776506.299285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776506.299288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776506.299288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776506.299333:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776506.299336:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776506.299339:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776507.003292:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308061958, at 4308062016 00000100:00000040:1.0:1775776507.003326:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308061958, at 4308062016 00000100:00000010:0.0:1775776507.003329:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3a20. 00000100:00000010:1.0:1775776507.003350:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c30c0. 00000100:00000001:1.0:1775776507.195244:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776507.195250:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776507.195251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776507.195255:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776507.195259:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776507.195261:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776507.195263:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776507.195265:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776507.195267:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776507.323239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776507.323245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776507.323250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776507.323251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776507.323254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776507.323255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776507.323354:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776507.323361:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776507.323366:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775776508.219249:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776508.219250:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776508.219255:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776508.219257:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776508.219257:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776508.219260:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1775776508.219263:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:2.0:1775776508.219264:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776508.219267:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776508.219268:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776508.219273:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775776508.219276:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13390 00000100:00000001:1.0:1775776508.219277:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776508.219280:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776508.219284:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776508.219300:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776508.219308:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776508.219311:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776508.219314:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776508.219318:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776508.219320:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776508.219322:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776508.219325:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776508.219327:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776508.219329:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776508.219331:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776508.219333:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776508.219334:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776508.219337:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776508.219338:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776508.219339:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776508.219341:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776508.219343:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776508.219345:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776508.219348:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776508.219350:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776508.219353:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776508.219355:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776508.219357:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776508.219358:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776508.219365:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775776508.219368:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776508.219372:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776508.219373:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776508.219375:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776508.219378:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776508.219381:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776508.219388:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775776508.219393:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776508.219400:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710419584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776508.219407:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776508.219417:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776508.219421:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710419584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776508.219439:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776508.219442:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776508.219444:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776508.219445:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776508.219446:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776508.219449:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776508.219451:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13390 00000400:00000200:2.0:1775776508.219453:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776508.219455:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776508.219456:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776508.219458:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776508.219460:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776508.219461:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776508.219463:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776508.219464:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776508.219465:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776508.219466:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776508.219467:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776508.219469:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776508.219470:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776508.219472:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776508.219474:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776508.219475:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776508.219476:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776508.219478:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776508.219480:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776508.219481:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776508.219482:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776508.219487:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776508.219488:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776508.219489:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776508.219493:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776508.219495:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776508.219496:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776508.219499:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775776508.219501:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776508.219502:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776508.219503:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776508.219504:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776508.219505:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776508.219506:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776508.219509:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775776508.219511:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776508.219515:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710419712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776508.219521:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776508.219522:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776508.219524:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776508.219533:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1775776508.219534:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710419712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.219540:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1775776508.219549:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776508.219551:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776508.219556:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000040:0.0:1775776508.219557:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344aa00 x1862022710419584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776508.219558:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776508.219563:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776508.219574:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776508.219583:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13390 00000100:00000001:0.0:1775776508.219592:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776508.219595:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776508.219596:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:0.0:1775776508.219602:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:2.0:1775776508.219603:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 02000000:00000001:0.0:1775776508.219609:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776508.219617:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776508.219617:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775776508.219626:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710419584:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776508.219630:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776508.219635:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776508.219637:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776508.219642:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:2.0:1775776508.219643:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776508.219646:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776508.219647:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776508.219649:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000010:0.0:1775776508.219651:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4800. 00000020:00000001:2.0:1775776508.219652:0:191261:0:(genops.c:811:class_conn2export()) Process entered 02000000:00000001:0.0:1775776508.219656:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775776508.219657:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776508.219660:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776508.219662:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776508.219664:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:0.0:1775776508.219671:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000001:2.0:1775776508.219672:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000010:0.0:1775776508.219677:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000100:00000040:2.0:1775776508.219678:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776508.219687:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776508.219691:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000010:0.0:1775776508.219693:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000001:2.0:1775776508.219697:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776508.219702:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00000200:0.0:1775776508.219703:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710419584, portal 25 00000100:00000001:0.0:1775776508.219707:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776508.219709:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662545920 : -123823047005696 : ffff8f623344aa00) 00000100:00080000:2.0:1775776508.219709:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776508.219718:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776508.219720:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344aa00 x1862022710419584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775776508.219725:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776508.219734:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776508.219735:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776508.219743:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 02000000:00000001:2.0:1775776508.219744:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776508.219746:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000001:2.0:1775776508.219750:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000200:0.0:1775776508.219752:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710419584, offset 0 00000008:00100000:2.0:1775776508.219754:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776508.219757:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776508.219760:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776508.219761:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000010:2.0:1775776508.219766:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775776508.219769:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776508.219773:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1775776508.219774:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:2.0:1775776508.219777:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776508.219779:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776508.219781:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776508.219786:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776508.219787:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776508.219799:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000010:2.0:1775776508.219800:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000400:00000200:0.0:1775776508.219807:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:2.0:1775776508.219808:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776508.219813:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776508.219818:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00100000:2.0:1775776508.219819:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710419840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775776508.219828:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453830:453830:256:4294967295] 192.168.201.115@tcp LPNI seq info [453755:453755:8:4294967295] 00000400:00000200:0.0:1775776508.219835:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:2.0:1775776508.219846:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1775776508.219847:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:2.0:1775776508.219856:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775776508.219864:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000040:2.0:1775776508.219865:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710419840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:0.0:1775776508.219875:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000100:00000001:2.0:1775776508.219882:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776508.219885:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776508.219888:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776508.219889:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776508.219893:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776508.219903:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13400) 00000800:00000200:0.0:1775776508.219904:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776508.219910:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775776508.219916:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776508.219920:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1775776508.219924:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776508.219927:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776508.219933:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776508.219938:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776508.219942:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776508.219944:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776508.219948:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776508.219952:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344aa00 x1862022710419584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.219961:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776508.219963:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344aa00 x1862022710419584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776533 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.219969:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776508.219970:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776508.219975:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776508.219978:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776508.219981:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308067078 00000100:00000010:0.0:1775776508.219986:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3a20. 00000100:00000001:0.0:1775776508.219992:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776508.220001:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308067078 00000100:00000010:0.0:1775776508.220003:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3720. 00000100:00000001:0.0:1775776508.220005:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776508.220008:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776508.220010:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776508.220012:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776508.220015:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776508/real 1775776508] req@ffff8f623344aa00 x1862022710419584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776508.220022:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776508.220024:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776508.220025:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776508.220027:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776508.220028:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776508.220030:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776508.220032:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344aa00 x1862022710419584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776508.220038:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344aa00 x1862022710419584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.220043:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776508.220044:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776508.220046:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776508.220048:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776508.220051:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344aa00 x1862022710419584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.220055:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776508.220058:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344aa00 x1862022710419584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.220063:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776508.220065:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776508.220066:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776508.220067:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776508.220069:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776508.220070:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776508.220073:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776508.220076:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776508.220077:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776508.220080:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344aa00 x1862022710419584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776508.220085:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710419584:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776508.220089:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776508.220091:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776508.220092:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344aa00 x1862022710419584/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.220096:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776508.220099:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776508.220101:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775776508.220103:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776508.220105:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776508.220106:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776508.220108:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776508.220110:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4cc00. 02000000:00000001:0.0:1775776508.220113:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776508.220115:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776508.220117:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344aa00. 00000100:00000001:0.0:1775776508.220120:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776508.220121:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776508.220123:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776508.220129:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776508.220131:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776508.220187:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776508.220189:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776508.220214:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776508.220217:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233448380 x1862022710419712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.220223:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776508.220224:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776508.220226:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776508.220228:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776508.220229:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776508.220231:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233448380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710419712:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776508.220233:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776508.220234:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776508.220236:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776508.220237:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776508.220239:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775776508.220241:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776508.220243:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776508.220245:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776508.220247:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776508.220250:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710419712, portal 10 00000100:00000001:0.0:1775776508.220251:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776508.220252:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662536064 : -123823047015552 : ffff8f6233448380) 00000100:00000040:0.0:1775776508.220257:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233448380 x1862022710419712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.220262:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776508.220264:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776508.220266:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776508.220267:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710419712, offset 0 00000400:00000200:0.0:1775776508.220270:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776508.220272:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776508.220275:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776508.220277:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776508.220279:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776508.220280:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776508.220282:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776508.220284:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453831:453831:256:4294967295] 192.168.201.115@tcp LPNI seq info [453756:453756:8:4294967295] 00000400:00000200:0.0:1775776508.220304:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776508.220311:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776508.220315:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776508.220319:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776508.220323:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776508.220326:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775776508.220330:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776508.220333:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1775776508.220337:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776508.220339:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776508.220343:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776508.220347:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776508.220350:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776508.220352:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776508.220355:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776508.220358:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233448380 x1862022710419712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.220368:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776508.220370:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448380 x1862022710419712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776533 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.220380:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776508.220381:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776508.220384:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776508.220387:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776508.220390:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776508.220392:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308063244 00000100:00000001:0.0:1775776508.220394:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308067078 00000100:00000001:0.0:1775776508.220396:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308063244 00000100:00000001:0.0:1775776508.220397:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308067078 00000100:00000001:0.0:1775776508.220399:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776508.220400:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776508.220401:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776508.220404:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776508/real 1775776508] req@ffff8f6233448380 x1862022710419712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776508.220409:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776508.220410:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776508.220412:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776508.220413:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776508.220414:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776508.220416:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776508.220418:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233448380 x1862022710419712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776508.220423:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233448380 x1862022710419712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.220428:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776508.220429:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776508.220430:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776508.220431:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776508.220433:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233448380 x1862022710419712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.220438:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776508.220441:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233448380 x1862022710419712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.220445:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776508.220447:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776508.220448:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776508.220448:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776508.220450:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776508.220454:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776508.220456:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776508.220458:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776508.220459:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776508.220461:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233448380 x1862022710419712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776508.220466:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233448380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710419712:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776508.220471:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776508.220475:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449f80 x1862022710419840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.220482:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776508.220484:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776508.220486:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776508.220488:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776508.220490:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776508.220493:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710419840:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776508.220497:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776508.220500:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776508.220502:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776508.220504:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776508.220507:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776508.220509:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776508.220514:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776508.220516:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776508.220520:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776508.220523:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710419840, portal 10 00000100:00000001:0.0:1775776508.220525:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776508.220526:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662543232 : -123823047008384 : ffff8f6233449f80) 00000100:00000040:0.0:1775776508.220532:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449f80 x1862022710419840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.220537:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776508.220539:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776508.220540:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776508.220542:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710419840, offset 0 00000400:00000200:0.0:1775776508.220545:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776508.220548:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776508.220550:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776508.220552:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776508.220554:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776508.220556:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776508.220558:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776508.220560:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453832:453832:256:4294967295] 192.168.201.115@tcp LPNI seq info [453757:453757:8:4294967295] 00000400:00000200:0.0:1775776508.220563:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776508.220567:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776508.220570:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776508.220572:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000800:00000200:0.0:1775776508.220575:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776508.220577:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1775776508.220580:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776508.220581:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1775776508.220584:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776508.220586:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:0.0:1775776508.220588:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776508.220591:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776508.220593:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:0.0:1775776508.220594:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:0.0:1775776508.220595:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776508.220597:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449f80 x1862022710419840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.220603:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776508.220605:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449f80 x1862022710419840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776533 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.220610:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776508.220611:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776508.220613:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776508.220615:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776508.220617:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776508.220618:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308063244 00000100:00000001:0.0:1775776508.220620:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308067078 00000100:00000001:0.0:1775776508.220621:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308063244 00000100:00000001:0.0:1775776508.220623:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308067078 00000100:00000001:0.0:1775776508.220624:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776508.220626:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776508.220627:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776508.220629:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776508/real 1775776508] req@ffff8f6233449f80 x1862022710419840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776508.220634:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776508.220635:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776508.220636:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776508.220638:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776508.220639:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776508.220641:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776508.220643:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449f80 x1862022710419840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776508.220648:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449f80 x1862022710419840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.220652:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776508.220653:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776508.220654:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776508.220655:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776508.220657:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449f80 x1862022710419840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.220661:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776508.220663:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449f80 x1862022710419840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.220667:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776508.220668:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776508.220669:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776508.220670:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776508.220671:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776508.220672:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776508.220674:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776508.220676:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776508.220678:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776508.220680:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449f80 x1862022710419840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776508.220685:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710419840:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776508.220688:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776508.220689:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776508.220691:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233448380 x1862022710419712/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.220695:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776508.220697:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776508.220698:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775776508.220699:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776508.220700:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776508.220701:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776508.220702:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776508.220704:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4d400. 02000000:00000001:0.0:1775776508.220706:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776508.220707:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776508.220708:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233448380. 00000100:00000001:0.0:1775776508.220709:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776508.220710:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776508.220711:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776508.220713:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449f80 x1862022710419840/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776533 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776508.220718:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776508.220718:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776508.220719:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776508.220721:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776508.220722:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776508.220722:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776508.220723:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776508.220724:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4d800. 02000000:00000001:0.0:1775776508.220725:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776508.220726:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776508.220727:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449f80. 00000100:00000001:0.0:1775776508.220728:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776508.220729:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776508.220730:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776508.220733:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776508.220734:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776508.220735:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776508.220736:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776508.347241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776508.347247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776508.347253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776508.347254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776508.347258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776508.347260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776508.347325:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776508.347328:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776508.347331:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776509.243226:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776509.243232:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776509.243233:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776509.243245:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776509.243248:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776509.243251:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776509.243253:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776509.243256:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776509.243258:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776509.371214:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776509.371218:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776509.371222:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776509.371223:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776509.371226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776509.371227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776509.371306:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776509.371310:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776509.371314:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776510.267208:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776510.267214:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776510.267216:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776510.267220:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776510.267223:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776510.267226:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776510.267228:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776510.267230:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776510.267232:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776510.395306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776510.395315:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776510.395326:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776510.395327:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776510.395332:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776510.395334:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776510.395408:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776510.395413:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776510.395417:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776511.291281:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776511.291288:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776511.291290:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776511.291296:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776511.291300:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776511.291303:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776511.291306:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776511.291308:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776511.291311:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776511.419333:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776511.419342:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776511.419350:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776511.419355:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776511.419358:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776511.419362:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776511.419366:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776511.419370:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776511.419372:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776511.419374:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776511.419380:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776511.419381:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776511.419383:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776511.419385:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776511.419387:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776511.419388:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776511.419390:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776511.419393:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776511.419475:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776511.419481:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776511.419485:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776512.123231:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308067078, at 4308067136 00000100:00000040:0.0:1775776512.123240:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308067078, at 4308067136 00000100:00000010:1.0:1775776512.123252:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3720. 00000100:00000010:0.0:1775776512.123257:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3a20. 00000100:00000001:1.0:1775776512.315256:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776512.315262:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776512.315263:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776512.315267:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776512.315270:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776512.315273:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776512.315275:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776512.315278:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776512.315279:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776512.443287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776512.443293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776512.443298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776512.443299:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776512.443302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776512.443303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776512.443365:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776512.443369:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776512.443373:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776513.339275:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775776513.339275:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776513.339282:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776513.339284:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776513.339287:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1775776513.339290:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776513.339291:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776513.339294:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776513.339296:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776513.339298:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776513.339302:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776513.339303:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776513.339306:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776513.339309:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776513.339313:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13395 00000400:00000200:3.0:1775776513.339324:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776513.339333:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776513.339338:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776513.339341:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776513.339345:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776513.339347:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776513.339350:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776513.339354:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776513.339356:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776513.339359:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776513.339362:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776513.339365:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776513.339367:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776513.339370:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776513.339372:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776513.339374:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776513.339375:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776513.339378:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776513.339382:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776513.339384:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776513.339388:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776513.339391:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776513.339394:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776513.339397:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776513.339399:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776513.339411:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73100. 00000020:00000040:3.0:1775776513.339416:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776513.339421:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776513.339423:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776513.339427:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776513.339430:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776513.339434:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776513.339444:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000100:00000001:3.0:1775776513.339452:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776513.339462:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73100 x1862022710419968/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776513.339473:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776513.339479:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776513.339484:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f67e6552] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73100 x1862022710419968/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.339506:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776513.339509:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776513.339512:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776513.339514:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776513.339516:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776513.339519:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776513.339522:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13395 00000400:00000200:3.0:1775776513.339526:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776513.339528:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1775776513.339530:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:0.0:1775776513.339532:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776513.339533:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775776513.339535:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776513.339538:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776513.339541:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776513.339543:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73100 x1862022710419968/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776513.339545:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776513.339551:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776513.339555:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776513.339557:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:0.0:1775776513.339558:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776513.339561:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775776513.339561:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776513.339565:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 02000000:00000001:0.0:1775776513.339567:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1775776513.339568:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:0.0:1775776513.339570:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775776513.339572:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:0.0:1775776513.339573:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776513.339579:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710419968:192.168.201.115@tcp:250:kworker.0 00000100:00000040:3.0:1775776513.339580:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776513.339585:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776513.339587:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1775776513.339589:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776513.339591:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:0.0:1775776513.339592:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776513.339594:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776513.339595:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:0.0:1775776513.339598:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1775776513.339600:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776513.339603:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776513.339604:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:3.0:1775776513.339605:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776513.339607:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776513.339613:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776513.339613:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1775776513.339616:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 02000000:00000001:3.0:1775776513.339617:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776513.339621:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000010:0.0:1775776513.339624:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000008:00100000:3.0:1775776513.339626:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776513.339630:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775776513.339632:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710419968, portal 25 00000100:00000001:3.0:1775776513.339633:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776513.339637:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776513.339639:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849016576 : -123823860535040 : ffff8f6202c73100) 00000100:00000010:3.0:1775776513.339642:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000100:00000040:0.0:1775776513.339644:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73100 x1862022710419968/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776538 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775776513.339645:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776513.339650:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776513.339652:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776513.339656:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776513.339659:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776513.339662:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776513.339663:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776513.339666:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 02000000:00000010:3.0:1775776513.339667:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000400:00000010:0.0:1775776513.339669:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0bb0. 00000100:00000200:0.0:1775776513.339673:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710419968, offset 0 00000100:00000001:3.0:1775776513.339673:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776513.339682:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00100000:3.0:1775776513.339682:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710420096/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776513.339690:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1775776513.339693:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776513.339695:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776513.339699:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710420096/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776513.339701:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776513.339708:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776513.339714:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776513.339720:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776513.339723:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:3.0:1775776513.339723:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776513.339728:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775776513.339731:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453833:453833:256:4294967295] 192.168.201.115@tcp LPNI seq info [453758:453758:8:4294967295] 00000100:00080000:3.0:1775776513.339733:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776513.339736:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1775776513.339739:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00080000:3.0:1775776513.339739:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776513.339743:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776513.339747:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13395 00000400:00000200:0.0:1775776513.339752:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000400:00000200:3.0:1775776513.339756:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000200:0.0:1775776513.339760:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000400:00000200:3.0:1775776513.339762:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000800:00000010:0.0:1775776513.339766:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c300. 00000100:00080000:3.0:1775776513.339766:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776513.339771:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776513.339774:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.0:1775776513.339775:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776513.339779:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:0.1:1775776513.339782:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775776513.339783:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776513.339787:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776513.339789:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776513.339790:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776513.339792:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000100:00000001:0.0:1775776513.339800:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775776513.339804:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:0.0:1775776513.339805:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000001:3.0:1775776513.339808:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000001:0.0:1775776513.339810:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308072198 00000100:00000040:3.0:1775776513.339814:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000010:0.0:1775776513.339815:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3a20. 00000100:00000001:3.0:1775776513.339816:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776513.339821:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776513.339823:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:0.0:1775776513.339825:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000020:00000040:3.0:1775776513.339825:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776513.339831:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776513.339833:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776513.339834:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308072198 02000000:00000001:3.0:1775776513.339838:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000010:0.0:1775776513.339839:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3b40. 02000000:00000001:3.0:1775776513.339842:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776513.339844:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1775776513.339845:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776513.339848:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776513.339850:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1775776513.339851:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776513.339856:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776513.339857:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776513.339858:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776513.339862:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776513.339864:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000100:00000001:0.0:1775776513.339865:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1775776513.339867:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776513.339871:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776513.339873:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776513.339875:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1775776513.339876:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1775776513.339877:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776513.339879:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776513.339880:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000010:3.0:1775776513.339891:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000100:00000001:0.0:1775776513.339893:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776513.339899:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776513.339899:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776513.339902:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00100000:3.0:1775776513.339906:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710420224/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1775776513.339909:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70000 x1862022710420096/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776513.339922:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776513.339926:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776513.339926:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776513.339928:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776513.339931:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1775776513.339934:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710420224/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775776513.339935:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776513.339941:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776513.339950:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710420096:192.168.201.115@tcp:38:kworker.0 00000100:00000001:3.0:1775776513.339951:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776513.339956:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1775776513.339956:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1775776513.339960:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1775776513.339962:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 02000000:00000001:0.0:1775776513.339963:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776513.339966:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1775776513.339967:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1775776513.339971:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 02000000:00000010:0.0:1775776513.339973:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775776513.339976:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776513.339978:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13406) 00000400:00000200:0.0:1775776513.339982:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776513.339986:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d68. 00000400:00000010:0.0:1775776513.339991:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776513.339994:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710420096, portal 10 00000100:00000001:0.0:1775776513.339997:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776513.339998:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004032 : -123823860547584 : ffff8f6202c70000) 00000100:00000040:0.0:1775776513.340006:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70000 x1862022710420096/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776538 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776513.340012:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776513.340015:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776513.340017:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0330. 00000100:00000200:0.0:1775776513.340019:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710420096, offset 0 00000400:00000200:0.0:1775776513.340023:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776513.340027:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776513.340032:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776513.340035:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776513.340038:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776513.340041:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776513.340044:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776513.340048:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453834:453834:255:4294967295] 192.168.201.115@tcp LPNI seq info [453759:453759:7:4294967295] 00000400:00000200:0.0:1775776513.340060:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776513.340067:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776513.340071:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776513.340075:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cb00. 00000800:00000200:0.0:1775776513.340078:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776513.340082:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775776513.340084:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776513.340089:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776513.340092:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776513.340094:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308068363 00000100:00000001:0.0:1775776513.340096:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308072198 00000100:00000001:0.0:1775776513.340099:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308068363 00000100:00000001:0.0:1775776513.340100:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308072198 00000100:00000001:0.0:1775776513.340102:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776513.340105:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776513.340107:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776513.340109:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776513.340112:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776513.340113:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1775776513.340115:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776513.340117:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776513.340118:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776513.340122:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70e00 x1862022710420224/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776513.340172:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776513.340173:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776513.340175:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776513.340177:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776513.340179:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776513.340182:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710420224:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776513.340185:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776513.340187:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776513.340188:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776513.340190:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776513.340254:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776513.340257:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776513.340261:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776513.340263:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775776513.340267:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0908. 00000100:00000200:0.0:1775776513.340269:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710420224, portal 10 00000100:00000001:0.0:1775776513.340272:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776513.340273:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849007616 : -123823860544000 : ffff8f6202c70e00) 00000100:00000040:0.0:1775776513.340280:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70e00 x1862022710420224/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776538 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776513.340286:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776513.340288:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776513.340291:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0aa0. 00000100:00000200:0.0:1775776513.340293:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710420224, offset 0 00000400:00000200:0.0:1775776513.340297:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776513.340301:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776513.340305:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776513.340307:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776513.340310:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776513.340312:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776513.340314:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776513.340317:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453835:453835:254:4294967295] 192.168.201.115@tcp LPNI seq info [453760:453760:6:4294967295] 00000400:00000200:0.0:1775776513.340321:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776513.340327:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776513.340332:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776513.340336:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cf00. 00000800:00000200:0.0:1775776513.340340:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776513.340343:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775776513.340346:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776513.340349:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776513.340352:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776513.340354:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308068364 00000100:00000001:0.0:1775776513.340357:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308072198 00000100:00000001:0.0:1775776513.340359:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308068364 00000100:00000001:0.0:1775776513.340361:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308072198 00000100:00000001:0.0:1775776513.340363:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776513.340365:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776513.340368:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776513.340370:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776513.340374:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776513.340375:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1775776513.340377:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776513.340379:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776513.340380:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776513.340382:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776513.467328:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776513.467334:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776513.467341:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776513.467342:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776513.467345:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776513.467346:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776513.467394:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776513.467398:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776513.467403:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776514.363258:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776514.363267:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776514.363269:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776514.363276:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776514.363280:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776514.363291:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776514.363295:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776514.363299:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776514.363303:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776514.491292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776514.491298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776514.491303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776514.491304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776514.491306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776514.491307:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776514.491357:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776514.491361:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776514.491364:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776515.387285:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776515.387292:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776515.387294:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776515.387302:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776515.387307:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776515.387314:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776515.387318:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776515.387321:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776515.387324:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776515.515283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776515.515289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776515.515293:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776515.515294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776515.515297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776515.515298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776515.515352:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776515.515355:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776515.515358:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776516.411247:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776516.411264:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776516.411267:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776516.411275:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776516.411278:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776516.411282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776516.411284:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776516.411287:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776516.411289:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1775776516.411410:0:2401:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:2.0:1775776516.411458:0:2401:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1775776516.411470:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776516.411474:0:2401:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:2.0:1775776516.411479:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776516.411483:0:2401:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:2.0:1775776516.411490:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775776516.411493:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c300. 00000400:00000200:2.0:1775776516.411501:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775776516.411506:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775776516.411514:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0bb0 00000400:00000010:2.0:1775776516.411517:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0bb0. 00000100:00000001:2.0:1775776516.411523:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776516.411533:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c73100 x1862022710419968/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776538 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776516.411607:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776516.411611:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776516.411615:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1775776516.411635:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776516.411640:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73100 x1862022710419968/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776538 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776516.411912:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776513/real 1775776516] req@ffff8f6202c73100 x1862022710419968/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776538 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776516.411925:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776516.411927:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776516.411931:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776516.411935:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776516.411938:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776516.411941:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776516.411945:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73100 x1862022710419968/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776538 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776516.411954:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73100 x1862022710419968/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776538 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776516.411961:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776516.411963:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776516.411966:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776516.411968:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776516.411972:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73100 x1862022710419968/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776538 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776516.411978:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776516.411983:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73100 x1862022710419968/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776538 ref 2 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776516.411990:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776516.411993:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776516.411995:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776516.411996:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776516.411999:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776516.412002:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776516.412005:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776516.412010:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776516.412012:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776516.412017:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73100 x1862022710419968/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776538 ref 2 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776516.412024:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710419968:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776516.412029:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776516.412032:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776516.412035:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73100 x1862022710419968/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776538 ref 2 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776516.412041:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776516.412044:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776516.412045:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776516.412047:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776516.412048:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776516.412050:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776516.412090:0:2401:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1775776516.412095:0:2401:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1775776516.412099:0:2401:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:2.0:1775776516.412104:0:2401:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1775776516.412107:0:2401:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1775776516.412109:0:2401:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:2.0:1775776516.412112:0:2401:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1775776516.412115:0:2401:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35c00. 02000000:00000001:2.0:1775776516.412119:0:2401:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1775776516.412121:0:2401:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1775776516.412123:0:2401:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73100. 00000100:00000001:2.0:1775776516.412127:0:2401:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1775776516.412128:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776516.412130:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775776516.412194:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775776516.412197:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cb00. 00000400:00000200:2.0:1775776516.412200:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775776516.412204:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775776516.412208:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0330 00000400:00000010:2.0:1775776516.412210:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0330. 00000100:00000001:2.0:1775776516.412212:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776516.412216:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c70000 x1862022710420096/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776538 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776516.412232:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776516.412235:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70000 x1862022710420096/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776538 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776516.412242:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776516.412243:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775776516.412248:0:2401:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775776516.412251:0:2401:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cf00. 00000400:00000200:2.0:1775776516.412253:0:2401:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775776516.412256:0:2401:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775776516.412267:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0aa0 00000400:00000010:2.0:1775776516.412269:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0aa0. 00000100:00000001:2.0:1775776516.412271:0:2401:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776516.412276:0:2401:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c70e00 x1862022710420224/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776538 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776516.412283:0:2401:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776516.412286:0:2401:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70e00 x1862022710420224/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776538 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776516.412293:0:2401:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776516.412294:0:2401:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776516.412624:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776516.412627:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776516.412629:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776516.412634:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776513/real 1775776516] req@ffff8f6202c70e00 x1862022710420224/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776516.412643:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776516.412644:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776516.412646:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775776516.412649:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0908 00000400:00000010:0.0:1775776516.412651:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0908. 00000100:00000001:0.0:1775776516.412653:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776516.412655:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70e00 x1862022710420224/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776516.412663:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70e00 x1862022710420224/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776516.412669:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776516.412670:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776516.412673:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776516.412674:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776516.412677:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70e00 x1862022710420224/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776516.412684:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776516.412688:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70e00 x1862022710420224/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776538 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776516.412695:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776516.412697:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776516.412699:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776516.412701:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776516.412703:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776516.412705:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776516.412708:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776516.412713:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776516.412716:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776516.412720:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70e00 x1862022710420224/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776538 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776516.412728:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710420224:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776516.412732:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776516.412736:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776513/real 1775776516] req@ffff8f6202c70000 x1862022710420096/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776516.412744:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776516.412746:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776516.412747:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d68. 00000400:00000200:0.0:1775776516.412750:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776516.412751:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776516.412754:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776516.412757:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70000 x1862022710420096/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776516.412765:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70000 x1862022710420096/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776516.412771:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776516.412774:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776516.412775:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776516.412776:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776516.412780:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70000 x1862022710420096/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776516.412785:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776516.412788:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70000 x1862022710420096/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776538 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776516.412794:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776516.412797:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776516.412798:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776516.412799:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776516.412801:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776516.412806:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776516.412808:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776516.412812:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776516.412814:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776516.412817:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70000 x1862022710420096/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776538 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776516.412824:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710420096:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776516.412828:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776516.412829:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776516.412832:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70e00 x1862022710420224/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776538 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776516.412837:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776516.412839:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776516.412841:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776516.412845:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776516.412846:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776516.412848:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776516.412850:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776516.412852:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36c00. 02000000:00000001:0.0:1775776516.412855:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776516.412856:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776516.412857:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70e00. 00000100:00000001:0.0:1775776516.412861:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776516.412862:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776516.412863:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776516.412866:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70000 x1862022710420096/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776538 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776516.412871:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776516.412873:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776516.412874:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775776516.412876:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776516.412877:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776516.412878:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776516.412879:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776516.412880:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34000. 02000000:00000001:0.0:1775776516.412882:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776516.412883:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776516.412884:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70000. 00000100:00000001:0.0:1775776516.412886:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776516.412888:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776516.412890:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776516.412900:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776516.412902:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776516.412904:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776516.412905:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776516.539333:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776516.539343:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776516.539346:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776516.539350:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776516.539356:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776516.539359:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776516.539361:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776516.539364:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776516.539369:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776516.539371:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776516.539374:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776516.539376:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776516.539438:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776516.539442:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776516.539445:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776517.243248:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308072198, at 4308072256 00000100:00000040:1.0:1775776517.243248:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308072198, at 4308072256 00000100:00000010:0.0:1775776517.243265:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3a20. 00000100:00000010:1.0:1775776517.243272:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3b40. 00000100:00000001:1.0:1775776517.435281:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776517.435289:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776517.435292:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776517.435299:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776517.435304:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776517.435310:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776517.435313:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776517.435316:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776517.435319:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776517.563287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776517.563294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776517.563300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776517.563301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776517.563306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776517.563307:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776517.563372:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776517.563377:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776517.563380:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776518.459257:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776518.459263:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776518.459266:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776518.459280:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776518.459285:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776518.459290:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776518.459294:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776518.459297:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776518.459300:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776518.459309:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776518.459320:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776518.459325:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776518.459330:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776518.459342:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776518.459353:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13401 00000400:00000200:0.0:1775776518.459365:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776518.459372:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776518.459376:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776518.459379:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776518.459383:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776518.459386:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776518.459387:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776518.459391:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776518.459393:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776518.459395:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776518.459398:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776518.459399:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776518.459401:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776518.459404:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776518.459406:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776518.459407:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776518.459409:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776518.459412:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776518.459416:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776518.459418:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776518.459422:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776518.459426:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776518.459429:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776518.459432:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776518.459435:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776518.459442:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062a00. 00000020:00000040:0.0:1775776518.459446:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776518.459451:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776518.459453:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776518.459456:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776518.459459:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776518.459465:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776518.459486:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4800. 00000100:00000001:0.0:1775776518.459494:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776518.459507:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062a00 x1862022710420352/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776518.459526:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776518.459532:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776518.459536:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009753f9f6] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062a00 x1862022710420352/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776518.459571:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776518.459575:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776518.459583:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776518.459585:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776518.459586:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776518.459591:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776518.459593:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13401 00000400:00000200:0.0:1775776518.459597:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776518.459600:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776518.459602:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776518.459605:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776518.459607:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776518.459609:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776518.459611:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776518.459613:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776518.459614:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776518.459616:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776518.459618:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775776518.459620:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775776518.459622:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775776518.459625:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776518.459627:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776518.459628:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776518.459630:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776518.459633:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775776518.459635:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776518.459637:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776518.459638:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776518.459645:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776518.459647:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776518.459649:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776518.459653:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776518.459656:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776518.459658:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776518.459661:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062d80. 00000020:00000040:0.0:1775776518.459663:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776518.459666:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776518.459667:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776518.459668:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776518.459670:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776518.459671:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776518.459674:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6000. 00000100:00000001:0.0:1775776518.459678:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776518.459682:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062d80 x1862022710420480/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776518.459690:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776518.459691:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776518.459695:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009965f80b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062d80 x1862022710420480/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776518.459707:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776518.459710:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776518.459714:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776518.459715:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776518.459716:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776518.459719:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776518.459722:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13401 00000400:00000200:0.0:1775776518.459725:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776518.459728:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776518.459731:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776518.459733:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776518.459735:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776518.459737:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776518.459739:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776518.459741:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776518.459742:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776518.459743:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776518.459745:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775776518.459747:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775776518.459749:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775776518.459751:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776518.459753:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776518.459755:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776518.459757:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776518.459759:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775776518.459761:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776518.459763:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776518.459766:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776518.459769:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776518.459783:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776518.459784:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776518.459787:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776518.459789:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776518.459791:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776518.459794:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062680. 00000020:00000040:0.0:1775776518.459796:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776518.459799:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776518.459800:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776518.459801:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776518.459803:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776518.459804:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776518.459808:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7800. 00000100:00000001:0.0:1775776518.459812:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776518.459816:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062680 x1862022710420608/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776518.459822:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776518.459824:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776518.459828:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000082c7985e] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062680 x1862022710420608/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776518.459835:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776518.459837:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776518.459841:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776518.459844:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776518.459848:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776518.459853:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13411) 00000100:00000001:1.0:1775776518.460203:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776518.460208:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776518.460212:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776518.460219:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e062a00 x1862022710420352/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.460229:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776518.460230:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776518.460235:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776518.460237:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776518.460240:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776518.460244:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e062a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710420352:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776518.460249:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776518.460254:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776518.460257:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776518.460260:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776518.460267:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776518.460269:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776518.460276:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776518.460280:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776518.460288:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776518.460297:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710420352, portal 25 00000100:00000001:1.0:1775776518.460300:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776518.460301:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574563840 : -123823134987776 : ffff8f622e062a00) 00000100:00000040:1.0:1775776518.460307:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e062a00 x1862022710420352/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776543 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.460315:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776518.460318:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776518.460321:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776518.460325:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710420352, offset 0 00000400:00000200:1.0:1775776518.460332:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776518.460341:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776518.460349:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776518.460355:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776518.460360:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776518.460364:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776518.460368:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776518.460374:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453836:453836:256:4294967295] 192.168.201.115@tcp LPNI seq info [453761:453761:8:4294967295] 00000400:00000200:1.0:1775776518.460379:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776518.460389:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776518.460396:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776518.460404:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776518.460413:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776518.460417:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775776518.460424:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776518.460427:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775776518.460431:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776518.460434:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776518.460441:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776518.460445:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776518.460451:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776518.460454:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776518.460461:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776518.460464:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e062a00 x1862022710420352/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776543 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.460487:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776518.460490:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e062a00 x1862022710420352/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776543 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.460497:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776518.460499:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776518.460505:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776518.460509:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776518.460513:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308077319 00000100:00000010:1.0:1775776518.460518:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e20c0. 00000100:00000001:1.0:1775776518.460527:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776518.460534:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308077319 00000100:00000010:1.0:1775776518.460537:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2b40. 00000100:00000001:1.0:1775776518.460540:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776518.460543:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776518.460546:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776518.460549:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776518.460554:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776518/real 1775776518] req@ffff8f622e062a00 x1862022710420352/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776518.460564:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776518.460566:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776518.460568:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776518.460571:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776518.460572:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776518.460575:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776518.460578:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e062a00 x1862022710420352/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776518.460585:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e062a00 x1862022710420352/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.460596:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776518.460598:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776518.460600:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776518.460602:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776518.460605:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e062a00 x1862022710420352/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.460611:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776518.460616:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e062a00 x1862022710420352/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.460622:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776518.460625:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776518.460627:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776518.460628:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776518.460631:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776518.460633:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776518.460636:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776518.460641:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776518.460643:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776518.460647:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e062a00 x1862022710420352/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776518.460654:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e062a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710420352:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776518.460659:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776518.460662:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e062d80 x1862022710420480/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.460669:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776518.460670:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776518.460672:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776518.460674:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776518.460676:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776518.460678:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e062d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710420480:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776518.460681:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776518.460682:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776518.460684:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776518.460686:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776518.460689:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776518.460690:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776518.460694:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776518.460696:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776518.460700:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776518.460703:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710420480, portal 10 00000100:00000001:1.0:1775776518.460704:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776518.460706:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574564736 : -123823134986880 : ffff8f622e062d80) 00000100:00000040:1.0:1775776518.460713:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e062d80 x1862022710420480/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776543 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.460719:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776518.460721:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776518.460723:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776518.460725:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710420480, offset 0 00000400:00000200:1.0:1775776518.460729:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776518.460734:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776518.460737:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776518.460740:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776518.460742:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776518.460744:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776518.460747:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776518.460750:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453837:453837:256:4294967295] 192.168.201.115@tcp LPNI seq info [453762:453762:8:4294967295] 00000400:00000200:1.0:1775776518.460754:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776518.460760:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776518.460764:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776518.460767:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776518.460772:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776518.460774:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775776518.460778:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776518.460781:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775776518.460784:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776518.460786:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776518.460789:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776518.460792:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776518.463170:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776518.463190:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776518.463203:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776518.463214:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e062d80 x1862022710420480/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776543 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.463229:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776518.463233:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e062d80 x1862022710420480/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776543 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.463239:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776518.463242:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776518.463252:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776518.463258:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776518.463265:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776518.463269:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308073487 00000100:00000001:1.0:1775776518.463274:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308077319 00000100:00000001:1.0:1775776518.463276:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308073487 00000100:00000001:1.0:1775776518.463279:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308077319 00000100:00000001:1.0:1775776518.463281:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776518.463284:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776518.463288:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776518.463292:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776518/real 1775776518] req@ffff8f622e062d80 x1862022710420480/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776518.463303:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776518.463305:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776518.463307:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776518.463310:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776518.463312:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776518.463314:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776518.463317:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e062d80 x1862022710420480/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776518.463324:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e062d80 x1862022710420480/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.463330:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776518.463331:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776518.463334:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776518.463335:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776518.463339:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e062d80 x1862022710420480/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.463345:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776518.463350:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e062d80 x1862022710420480/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.463357:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776518.463360:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776518.463361:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776518.463362:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776518.463365:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776518.463373:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776518.463376:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776518.463380:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776518.463383:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776518.463387:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e062d80 x1862022710420480/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776518.463394:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e062d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710420480:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776518.463399:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776518.463403:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e062680 x1862022710420608/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.463409:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776518.463411:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776518.463415:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776518.463419:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776518.463422:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776518.463425:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e062680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710420608:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776518.463428:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776518.463433:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776518.463436:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776518.463439:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776518.463450:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776518.463452:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776518.463457:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776518.463461:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776518.463467:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776518.463480:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710420608, portal 10 00000100:00000001:1.0:1775776518.463483:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776518.463485:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574562944 : -123823134988672 : ffff8f622e062680) 00000100:00000040:1.0:1775776518.463491:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e062680 x1862022710420608/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776543 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.463495:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776518.463497:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776518.463499:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776518.463501:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710420608, offset 0 00000400:00000200:1.0:1775776518.463506:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776518.463512:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776518.463518:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776518.463523:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776518.463527:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776518.463530:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776518.463533:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776518.463536:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453838:453838:256:4294967295] 192.168.201.115@tcp LPNI seq info [453763:453763:8:4294967295] 00000400:00000200:1.0:1775776518.463539:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776518.463545:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776518.463550:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776518.463555:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776518.463561:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776518.463564:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775776518.463569:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776518.463571:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775776518.463574:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776518.463576:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776518.463580:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776518.463583:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776518.463586:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776518.463588:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776518.463591:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776518.463594:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e062680 x1862022710420608/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776543 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.463601:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776518.463604:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e062680 x1862022710420608/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776543 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.463609:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776518.463611:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776518.463613:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776518.463616:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776518.463619:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776518.463620:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308073487 00000100:00000001:1.0:1775776518.463623:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308077319 00000100:00000001:1.0:1775776518.463625:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308073487 00000100:00000001:1.0:1775776518.463627:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308077319 00000100:00000001:1.0:1775776518.463629:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776518.463631:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776518.463633:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776518.463636:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776518/real 1775776518] req@ffff8f622e062680 x1862022710420608/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776518.463644:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776518.463645:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776518.463647:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776518.463649:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776518.463651:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776518.463652:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776518.463654:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e062680 x1862022710420608/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776518.463660:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e062680 x1862022710420608/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.463664:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776518.463665:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776518.463666:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776518.463667:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776518.463669:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e062680 x1862022710420608/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.463672:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776518.463674:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e062680 x1862022710420608/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.463678:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776518.463679:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776518.463680:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776518.463681:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776518.463682:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776518.463683:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776518.463685:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776518.463687:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776518.463688:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776518.463690:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e062680 x1862022710420608/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776518.463695:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e062680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710420608:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776518.463699:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776518.463701:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776518.463710:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e062a00 x1862022710420352/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.463718:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776518.463722:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776518.463726:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776518.463729:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776518.463731:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776518.463733:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776518.463735:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776518.463738:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:1.0:1775776518.463743:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776518.463745:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776518.463747:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e062a00. 00000100:00000001:1.0:1775776518.463750:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776518.463751:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776518.463753:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776518.463756:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e062d80 x1862022710420480/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.463761:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776518.463762:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776518.463764:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776518.463766:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776518.463767:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776518.463768:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776518.463770:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776518.463772:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:1.0:1775776518.463774:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776518.463775:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776518.463776:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e062d80. 00000100:00000001:1.0:1775776518.463778:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776518.463779:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776518.463781:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776518.463784:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e062680 x1862022710420608/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776543 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776518.463790:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776518.463791:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776518.463793:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776518.463795:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776518.463796:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776518.463797:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776518.463800:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776518.463801:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:1.0:1775776518.463803:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776518.463804:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776518.463805:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e062680. 00000100:00000001:1.0:1775776518.463807:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776518.463808:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776518.463810:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776518.463819:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776518.463821:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776518.463824:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776518.463826:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776518.587240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776518.587245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776518.587248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776518.587249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776518.587252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776518.587252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776518.587319:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776518.587323:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776518.587326:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776519.483237:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776519.483243:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776519.483245:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776519.483250:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776519.483253:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776519.483256:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776519.483259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776519.483261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776519.483264:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776519.611281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776519.611286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776519.611291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776519.611292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776519.611296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776519.611297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776519.611353:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776519.611357:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776519.611360:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776520.507830:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776520.507838:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776520.507839:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776520.507844:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776520.507848:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776520.507852:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776520.507854:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776520.507856:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776520.507859:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776520.635246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776520.635250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776520.635253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776520.635254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776520.635257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776520.635258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776520.635314:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776520.635317:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776520.635320:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776521.531265:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776521.531272:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776521.531275:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776521.531280:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776521.531284:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776521.531288:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776521.531291:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776521.531293:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776521.531295:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776521.659208:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776521.659213:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776521.659216:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776521.659219:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776521.659221:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776521.659224:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776521.659227:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776521.659229:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776521.659230:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776521.659233:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776521.659236:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776521.659238:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776521.659239:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776521.659241:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776521.659242:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776521.659243:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776521.659245:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776521.659247:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776521.659315:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776521.659318:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776521.659321:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776522.363229:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308077319, at 4308077376 00000100:00000040:0.0:1775776522.363230:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308077319, at 4308077376 00000100:00000010:1.0:1775776522.363246:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2b40. 00000100:00000010:0.0:1775776522.363248:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e20c0. 00000100:00000001:1.0:1775776522.555281:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776522.555290:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776522.555292:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776522.555298:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776522.555303:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776522.555307:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776522.555311:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776522.555313:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776522.555316:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776522.683293:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776522.683301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776522.683308:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776522.683309:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776522.683312:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776522.683314:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776522.683648:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776522.683652:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776522.683656:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776523.579247:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776523.579256:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776523.579259:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776523.579263:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776523.579267:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776523.579269:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776523.579273:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1775776523.579275:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13406 00000100:00000001:2.0:1775776523.579275:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776523.579281:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776523.579283:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776523.579286:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775776523.579287:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:2.0:1775776523.579288:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776523.579290:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776523.579293:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776523.579295:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776523.579299:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776523.579303:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776523.579307:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776523.579309:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776523.579312:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776523.579315:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776523.579317:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776523.579320:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776523.579322:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776523.579324:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776523.579327:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776523.579330:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776523.579332:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776523.579333:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776523.579335:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776523.579338:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776523.579340:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776523.579344:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776523.579347:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776523.579349:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776523.579352:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776523.579364:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776523.579366:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776523.579375:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063800. 00000020:00000040:0.0:1775776523.579378:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776523.579381:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776523.579383:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776523.579386:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776523.579388:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776523.579391:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776523.579398:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000001:0.0:1775776523.579404:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776523.579412:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063800 x1862022710420736/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776523.579426:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776523.579430:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776523.579435:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000015d81d25] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063800 x1862022710420736/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776523.579454:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776523.579457:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776523.579460:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776523.579461:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776523.579463:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776523.579465:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776523.579467:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13406 00000400:00000200:0.0:1775776523.579472:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776523.579475:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776523.579478:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776523.579478:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1775776523.579483:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776523.579484:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776523.579487:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1775776523.579488:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776523.579491:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776523.579494:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:1.0:1775776523.579495:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e063800 x1862022710420736/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775776523.579497:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776523.579499:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776523.579502:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776523.579505:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775776523.579509:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776523.579510:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776523.579512:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776523.579516:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776523.579518:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775776523.579520:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776523.579522:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776523.579531:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776523.579533:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1775776523.579535:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e063800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710420736:192.168.201.115@tcp:250:kworker.0 00000100:00000040:0.0:1775776523.579537:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776523.579542:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:1.0:1775776523.579544:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:0.0:1775776523.579545:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:1.0:1775776523.579548:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1775776523.579549:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000001:1.0:1775776523.579551:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776523.579553:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776523.579555:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1775776523.579556:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:1.0:1775776523.579561:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:0.0:1775776523.579562:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776523.579565:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776523.579566:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776523.579571:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776523.579576:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1775776523.579576:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000008:00000001:0.0:1775776523.579581:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776523.579582:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000001:0.0:1775776523.579583:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776523.579588:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060380. 00000020:00000040:0.0:1775776523.579591:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000400:00000010:1.0:1775776523.579591:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 02000000:00000001:0.0:1775776523.579595:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776523.579597:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776523.579599:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:1.0:1775776523.579599:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710420736, portal 25 02000000:00000001:0.0:1775776523.579600:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776523.579604:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776523.579604:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776523.579607:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574567424 : -123823134984192 : ffff8f622e063800) 02000000:00000010:0.0:1775776523.579610:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6400. 00000100:00000040:1.0:1775776523.579614:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e063800 x1862022710420736/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776548 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776523.579620:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776523.579624:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1775776523.579630:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060380 x1862022710420864/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1775776523.579630:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776523.579633:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776523.579638:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710420736, offset 0 00000100:00000001:0.0:1775776523.579643:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776523.579646:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776523.579650:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000040:0.0:1775776523.579654:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000614ace82] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060380 x1862022710420864/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776523.579661:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776523.579667:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776523.579670:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00080000:0.0:1775776523.579672:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776523.579677:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:1.0:1775776523.579678:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1775776523.579679:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776523.579682:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775776523.579684:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:0.0:1775776523.579686:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775776523.579690:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00080000:0.0:1775776523.579691:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13406 00000400:00000200:1.0:1775776523.579696:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776523.579700:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453839:453839:256:4294967295] 192.168.201.115@tcp LPNI seq info [453764:453764:8:4294967295] 00000400:00000200:1.0:1775776523.579705:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776523.579714:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000400:00000200:0.0:1775776523.579720:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000200:1.0:1775776523.579722:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776523.579727:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000800:00000010:1.0:1775776523.579728:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000100:00080000:0.0:1775776523.579730:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776523.579734:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776523.579736:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.0:1775776523.579738:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:0.0:1775776523.579741:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:1.0:1775776523.579743:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000100:00000001:0.0:1775776523.579744:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776523.579748:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776523.579750:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776523.579754:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:1.1:1775776523.579754:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776523.579759:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000020:00000040:0.0:1775776523.579762:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000800:00000100:1.0:1775776523.579764:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000020:00000001:0.0:1775776523.579765:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000010:1.0:1775776523.579767:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000020:00000001:0.0:1775776523.579768:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775776523.579774:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776523.579774:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1775776523.579777:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776523.579779:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776523.579781:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776523.579783:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775776523.579788:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776523.579793:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776523.579793:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 02000000:00000001:0.0:1775776523.579797:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776523.579801:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775776523.579801:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 02000000:00000001:0.0:1775776523.579804:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776523.579805:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:0.0:1775776523.579806:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776523.579809:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775776523.579811:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000008:00000001:0.0:1775776523.579812:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776523.579815:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776523.579818:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e063800 x1862022710420736/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776548 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775776523.579821:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062680. 00000020:00000040:0.0:1775776523.579825:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000100:00000001:1.0:1775776523.579831:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1775776523.579836:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1775776523.579836:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e063800 x1862022710420736/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776548 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775776523.579838:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776523.579842:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776523.579845:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776523.579848:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776523.579850:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776523.579853:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4400. 00000100:00000001:1.0:1775776523.579853:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776523.579858:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776523.579858:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776523.579865:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00100000:0.0:1775776523.579866:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062680 x1862022710420992/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776523.579868:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308082438 00000100:00000010:1.0:1775776523.579876:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2b40. 00000100:00000001:0.0:1775776523.579877:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776523.579880:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776523.579883:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000040:0.0:1775776523.579884:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000082c7985e] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062680 x1862022710420992/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.579889:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308082438 00000100:00000010:1.0:1775776523.579894:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2660. 00000100:00000001:0.0:1775776523.579899:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776523.579899:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000040:0.0:1775776523.579904:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776523.579905:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776523.579910:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776523.579911:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776523.579914:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000040:0.0:1775776523.579916:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776523.579921:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00100000:1.0:1775776523.579921:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776523/real 1775776523] req@ffff8f622e063800 x1862022710420736/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776523.579928:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13416) 00000400:00000001:1.0:1775776523.579934:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776523.579936:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776523.579938:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776523.579941:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776523.579943:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776523.579946:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776523.579949:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e063800 x1862022710420736/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776523.579957:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e063800 x1862022710420736/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.579964:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776523.579966:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776523.579968:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776523.579970:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776523.579973:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e063800 x1862022710420736/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.579980:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776523.579984:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e063800 x1862022710420736/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.579991:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776523.579994:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776523.579995:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776523.579996:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776523.579999:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776523.580000:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776523.580003:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776523.580006:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776523.580009:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776523.580012:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e063800 x1862022710420736/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776523.580033:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e063800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710420736:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776523.580038:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776523.580039:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776523.580042:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e063800 x1862022710420736/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.580048:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776523.580051:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776523.580054:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776523.580057:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776523.580058:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776523.580060:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776523.580062:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776523.580064:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:1.0:1775776523.580068:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776523.580069:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776523.580071:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e063800. 00000100:00000001:1.0:1775776523.580073:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776523.580075:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776523.580077:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776523.580083:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776523.580084:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776523.580086:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776523.580088:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776523.580089:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776523.580092:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e060380 x1862022710420864/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.580099:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776523.580100:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776523.580102:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776523.580103:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776523.580105:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776523.580108:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e060380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710420864:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776523.580111:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776523.580113:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776523.580114:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776523.580116:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776523.580118:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776523.580120:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776523.580124:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776523.580126:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776523.580129:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776523.580132:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710420864, portal 10 00000100:00000001:1.0:1775776523.580151:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776523.580152:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574553984 : -123823134997632 : ffff8f622e060380) 00000100:00000040:1.0:1775776523.580158:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e060380 x1862022710420864/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776548 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.580164:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776523.580166:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776523.580169:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776523.580171:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710420864, offset 0 00000400:00000200:1.0:1775776523.580175:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776523.580178:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776523.580182:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776523.580184:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776523.580187:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776523.580194:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776523.580196:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776523.580199:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453840:453840:256:4294967295] 192.168.201.115@tcp LPNI seq info [453765:453765:8:4294967295] 00000400:00000200:1.0:1775776523.580203:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776523.580209:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776523.580213:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776523.580216:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776523.580219:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776523.580222:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775776523.580226:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776523.580228:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775776523.580232:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776523.580234:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776523.580236:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776523.580239:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776523.580242:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776523.580243:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776523.580246:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776523.580249:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e060380 x1862022710420864/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776548 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.580262:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776523.580265:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e060380 x1862022710420864/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776548 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.580272:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776523.580274:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776523.580276:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776523.580279:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776523.580283:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776523.580285:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308078604 00000100:00000001:1.0:1775776523.580288:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308082438 00000100:00000001:1.0:1775776523.580290:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308078604 00000100:00000001:1.0:1775776523.580292:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308082438 00000100:00000001:1.0:1775776523.580294:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776523.580295:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776523.580297:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776523.580300:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776523/real 1775776523] req@ffff8f622e060380 x1862022710420864/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776523.580309:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776523.580310:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776523.580311:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776523.580313:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776523.580315:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776523.580317:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776523.580320:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e060380 x1862022710420864/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776523.580328:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e060380 x1862022710420864/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.580335:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776523.580336:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776523.580338:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776523.580339:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776523.580342:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e060380 x1862022710420864/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.580349:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776523.580353:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e060380 x1862022710420864/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.580359:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776523.580361:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776523.580363:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776523.580364:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776523.580366:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776523.580370:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776523.580372:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776523.580375:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776523.580377:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776523.580381:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e060380 x1862022710420864/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776523.580388:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e060380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710420864:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776523.580392:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776523.580395:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e062680 x1862022710420992/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.580402:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776523.580403:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776523.580405:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776523.580406:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776523.580408:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776523.580411:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e062680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710420992:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776523.580413:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776523.580415:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776523.580416:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776523.580418:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776523.580421:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776523.580422:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776523.580425:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776523.580427:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776523.580430:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776523.580433:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710420992, portal 10 00000100:00000001:1.0:1775776523.580434:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776523.580436:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574562944 : -123823134988672 : ffff8f622e062680) 00000100:00000040:1.0:1775776523.580441:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e062680 x1862022710420992/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776548 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.580447:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776523.580449:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776523.580451:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776523.580453:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710420992, offset 0 00000400:00000200:1.0:1775776523.580456:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776523.580460:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776523.580463:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776523.580465:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776523.580467:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776523.580470:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776523.580472:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776523.580475:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453841:453841:256:4294967295] 192.168.201.115@tcp LPNI seq info [453766:453766:8:4294967295] 00000400:00000200:1.0:1775776523.580479:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776523.580485:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776523.580489:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776523.580491:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776523.580495:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776523.580497:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775776523.580501:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776523.580504:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775776523.580507:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776523.580509:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776523.580511:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776523.580515:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776523.580517:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776523.580518:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776523.580521:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776523.580523:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e062680 x1862022710420992/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776548 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.580530:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776523.580533:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e062680 x1862022710420992/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776548 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.580540:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776523.580541:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776523.580543:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776523.580546:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776523.580548:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776523.580550:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308078604 00000100:00000001:1.0:1775776523.580552:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308082438 00000100:00000001:1.0:1775776523.580554:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308078604 00000100:00000001:1.0:1775776523.580556:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308082438 00000100:00000001:1.0:1775776523.580558:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776523.580559:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776523.580561:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776523.580564:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776523/real 1775776523] req@ffff8f622e062680 x1862022710420992/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776523.580570:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776523.580572:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776523.580574:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776523.580576:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776523.580577:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776523.580579:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776523.580582:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e062680 x1862022710420992/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776523.580590:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e062680 x1862022710420992/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.580596:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776523.580597:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776523.580598:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776523.580599:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776523.580603:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e062680 x1862022710420992/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.580609:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776523.580612:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e062680 x1862022710420992/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.580619:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776523.580620:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776523.580622:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776523.580623:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776523.580624:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776523.580626:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776523.580628:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776523.580631:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776523.580632:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776523.580636:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e062680 x1862022710420992/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776523.580650:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e062680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710420992:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776523.580653:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776523.580655:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776523.580657:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e060380 x1862022710420864/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.580664:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776523.580665:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776523.580667:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776523.580669:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776523.580671:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776523.580672:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776523.580674:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776523.580676:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:1.0:1775776523.580678:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776523.580679:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776523.580681:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e060380. 00000100:00000001:1.0:1775776523.580683:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776523.580684:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776523.580686:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776523.580688:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e062680 x1862022710420992/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776548 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776523.580695:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776523.580696:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776523.580697:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776523.580699:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776523.580700:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776523.580701:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776523.580703:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776523.580704:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:1.0:1775776523.580706:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776523.580707:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776523.580708:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e062680. 00000100:00000001:1.0:1775776523.580711:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776523.580712:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776523.580714:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776523.580717:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776523.580718:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776523.580720:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776523.580722:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776523.707213:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776523.707217:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776523.707221:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776523.707222:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776523.707225:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776523.707226:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776523.707280:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776523.707284:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776523.707288:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776524.603262:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776524.603269:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776524.603270:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776524.603276:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776524.603279:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776524.603283:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776524.603286:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776524.603288:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776524.603291:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776524.731240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776524.731244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776524.731248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776524.731249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776524.731252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776524.731253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776524.731306:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776524.731308:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776524.731311:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776525.627235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776525.627241:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776525.627243:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776525.627250:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776525.627253:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776525.627256:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776525.627259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776525.627261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776525.627263:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776525.755268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776525.755274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776525.755281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776525.755282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776525.755286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776525.755288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776525.755357:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776525.755361:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776525.755364:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776526.651272:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776526.651277:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776526.651279:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776526.651283:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776526.651286:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776526.651290:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776526.651293:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776526.651295:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776526.651297:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776526.779353:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776526.779361:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776526.779362:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776526.779365:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776526.779369:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776526.779371:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776526.779372:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776526.779374:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776526.779376:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776526.779377:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776526.779380:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776526.779381:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776526.779448:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776526.779452:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776526.779455:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776527.483250:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308082438, at 4308082496 00000100:00000040:1.0:1775776527.483250:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308082438, at 4308082496 00000100:00000010:0.0:1775776527.483266:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2b40. 00000800:00000200:3.0:1775776527.483266:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000100:00000010:1.0:1775776527.483268:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2660. 00000800:00000200:3.0:1775776527.483277:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:3.0:1775776527.483282:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:1.0:1775776527.483420:0:2405:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000001:1.0:1775776527.483426:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1775776527.483438:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1775776527.483441:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1775776527.483444:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776527.675274:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776527.675283:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776527.675285:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776527.675292:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776527.675296:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776527.675301:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776527.675304:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776527.675307:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776527.675310:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776527.804219:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776527.804232:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776527.804240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776527.804241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776527.804246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776527.804247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776527.804345:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776527.804351:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776527.804356:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776528.699264:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776528.699271:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776528.699273:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776528.699278:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776528.699282:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776528.699286:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776528.699290:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775776528.699291:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776528.699293:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776528.699297:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776528.699302:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776528.699305:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776528.699309:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776528.699312:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776528.699319:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13411 00000400:00000200:3.0:1775776528.699329:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776528.699335:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776528.699338:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776528.699340:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776528.699343:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776528.699345:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776528.699346:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776528.699349:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776528.699351:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776528.699353:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776528.699355:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776528.699356:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776528.699358:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776528.699360:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776528.699361:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776528.699362:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776528.699363:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776528.699366:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776528.699368:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776528.699369:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776528.699372:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776528.699375:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776528.699378:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776528.699381:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776528.699384:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776528.699395:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775776528.699399:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776528.699404:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776528.699406:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776528.699447:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776528.699451:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776528.699456:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776528.699466:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000100:00000001:3.0:1775776528.699474:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776528.699484:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710421120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776528.699493:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776528.699498:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776528.699502:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710421120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776528.699551:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776528.699555:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776528.699559:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776528.699560:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776528.699561:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776528.699564:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776528.699567:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13411 00000400:00000200:3.0:1775776528.699570:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776528.699573:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776528.699574:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776528.699576:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776528.699579:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776528.699580:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776528.699582:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776528.699584:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776528.699585:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776528.699586:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776528.699588:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776528.699589:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776528.699591:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776528.699593:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776528.699594:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776528.699595:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776528.699597:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776528.699599:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776528.699601:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776528.699603:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776528.699604:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776528.699609:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776528.699611:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776528.699612:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776528.699616:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776528.699618:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776528.699619:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776528.699620:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776528.699623:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775776528.699626:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000001:1.0:1775776528.699627:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1775776528.699629:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776528.699633:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776528.699634:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776528.699636:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776528.699638:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776528.699640:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776528.699644:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:3.0:1775776528.699650:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776528.699651:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71c00 x1862022710421120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.699665:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1775776528.699669:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710421248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776528.699670:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776528.699678:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776528.699683:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776528.699684:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776528.699689:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776528.699691:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776528.699694:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710421248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776528.699702:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710421120:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776528.699714:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776528.699719:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776528.699721:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776528.699724:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776528.699733:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776528.699736:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776528.699751:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776528.699755:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776528.699762:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776528.699770:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710421120, portal 25 00000100:00000001:3.0:1775776528.699773:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776528.699775:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1775776528.699776:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776528.699780:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849011200 : -123823860540416 : ffff8f6202c71c00) 00000100:00080000:3.0:1775776528.699782:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776528.699787:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776528.699790:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1775776528.699796:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71c00 x1862022710421120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776553 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776528.699798:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776528.699805:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13411 00000400:00000200:3.0:1775776528.699824:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776528.699826:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1775776528.699838:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000040:1.0:1775776528.699839:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00080000:3.0:1775776528.699845:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000010:1.0:1775776528.699846:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776528.699857:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710421120, offset 0 00000100:00000040:3.0:1775776528.699858:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776528.699865:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776528.699870:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776528.699876:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776528.699878:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776528.699884:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776528.699885:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776528.699888:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:1.0:1775776528.699891:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000040:3.0:1775776528.699894:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776528.699898:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776528.699902:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000020:00000001:3.0:1775776528.699902:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776528.699915:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1775776528.699915:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776528.699919:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776528.699921:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776528.699925:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776528.699954:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776528.699958:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776528.699963:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453842:453842:256:4294967295] 192.168.201.115@tcp LPNI seq info [453767:453767:8:4294967295] 00000400:00000200:1.0:1775776528.699970:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776528.699971:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776528.699982:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776528.699990:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776528.699998:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776528.700003:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000100:00080000:3.0:1775776528.700005:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776528.700010:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776528.700014:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:1.0:1775776528.700016:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:3.0:1775776528.700020:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.0:1775776528.700022:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 02000000:00000001:3.0:1775776528.700026:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776528.700030:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776528.700037:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000200:1.1:1775776528.700038:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000008:00000001:3.0:1775776528.700044:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776528.700049:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776528.700050:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000100:00000010:3.0:1775776528.700054:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775776528.700058:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000800:00000100:1.0:1775776528.700062:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 02000000:00000001:3.0:1775776528.700064:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000800:00000010:1.0:1775776528.700066:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 02000000:00000001:3.0:1775776528.700067:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776528.700073:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776528.700074:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1775776528.700076:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:3.0:1775776528.700077:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776528.700087:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000400:00000200:1.0:1775776528.700091:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1775776528.700094:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776528.700097:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776528.700101:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00100000:3.0:1775776528.700104:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710421376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776528.700116:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1775776528.700125:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:1.0:1775776528.700128:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71c00 x1862022710421120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776553 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776528.700128:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776528.700182:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710421376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.700220:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1775776528.700221:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776528.700227:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71c00 x1862022710421120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776553 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.700236:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776528.700238:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776528.700243:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776528.700247:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776528.700250:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308087559 00000100:00000040:3.0:1775776528.700258:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000010:1.0:1775776528.700260:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2660. 00000100:00000040:3.0:1775776528.700268:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776528.700272:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776528.700273:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000040:3.0:1775776528.700283:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776528.700289:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308087559 00000100:00000040:3.0:1775776528.700290:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13421) 00000100:00000010:1.0:1775776528.700294:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2de0. 00000100:00000001:1.0:1775776528.700299:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776528.700304:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776528.700307:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776528.700310:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776528.700314:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776528/real 1775776528] req@ffff8f6202c71c00 x1862022710421120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776528.700323:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776528.700325:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776528.700328:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776528.700330:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776528.700332:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776528.700334:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776528.700338:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71c00 x1862022710421120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776528.700353:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71c00 x1862022710421120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.700360:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776528.700362:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776528.700364:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776528.700366:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776528.700370:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71c00 x1862022710421120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.700377:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776528.700382:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71c00 x1862022710421120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.700390:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776528.700392:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776528.700394:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776528.700395:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776528.700397:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776528.700399:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776528.700402:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776528.700406:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776528.700409:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776528.700413:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71c00 x1862022710421120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776528.700422:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710421120:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776528.700427:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776528.700428:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776528.700431:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71c00 x1862022710421120/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.700444:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776528.700446:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776528.700449:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776528.700451:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776528.700454:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776528.700455:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776528.700458:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776528.700460:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35000. 02000000:00000001:1.0:1775776528.700464:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776528.700466:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776528.700468:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71c00. 00000100:00000001:1.0:1775776528.700471:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776528.700472:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776528.700474:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776528.700481:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776528.700483:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776528.700486:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776528.700487:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776528.700489:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776528.700493:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73480 x1862022710421248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.700500:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776528.700501:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776528.700503:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776528.700505:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776528.700507:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776528.700510:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710421248:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776528.700513:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776528.700516:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776528.700517:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776528.700519:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776528.700522:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776528.700524:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776528.700527:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776528.700530:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776528.700533:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776528.700536:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710421248, portal 10 00000100:00000001:1.0:1775776528.700538:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776528.700539:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849017472 : -123823860534144 : ffff8f6202c73480) 00000100:00000040:1.0:1775776528.700546:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73480 x1862022710421248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776553 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.700552:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776528.700554:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776528.700557:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776528.700559:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710421248, offset 0 00000400:00000200:1.0:1775776528.700563:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776528.700567:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776528.700571:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776528.700573:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776528.700576:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776528.700578:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776528.700581:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776528.700584:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453843:453843:256:4294967295] 192.168.201.115@tcp LPNI seq info [453768:453768:8:4294967295] 00000400:00000200:1.0:1775776528.700588:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776528.700594:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776528.700599:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776528.700602:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776528.700605:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776528.700608:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775776528.700613:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776528.700615:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775776528.700619:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776528.700621:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776528.700623:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776528.700626:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776528.700629:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776528.700631:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776528.700634:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776528.700637:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73480 x1862022710421248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776553 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.700644:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776528.700647:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73480 x1862022710421248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776553 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.700654:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776528.700656:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776528.700658:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776528.700662:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776528.700666:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776528.700668:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308083724 00000100:00000001:1.0:1775776528.700670:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308087559 00000100:00000001:1.0:1775776528.700673:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308083724 00000100:00000001:1.0:1775776528.700675:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308087559 00000100:00000001:1.0:1775776528.700677:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776528.700678:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776528.700680:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776528.700683:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776528/real 1775776528] req@ffff8f6202c73480 x1862022710421248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776528.700690:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776528.700691:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776528.700693:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776528.700695:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776528.700696:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776528.700698:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776528.700701:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73480 x1862022710421248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776528.700710:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73480 x1862022710421248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.700728:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776528.700729:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776528.700730:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776528.700732:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776528.700735:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73480 x1862022710421248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.700743:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776528.700746:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73480 x1862022710421248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.700754:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776528.700756:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776528.700758:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776528.700759:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776528.700761:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776528.700766:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776528.700769:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776528.700773:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776528.700775:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776528.700779:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73480 x1862022710421248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776528.700787:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710421248:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776528.700791:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776528.700794:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70380 x1862022710421376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.700801:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776528.700802:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776528.700805:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776528.700807:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776528.700808:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776528.700811:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710421376:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776528.700814:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776528.700816:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776528.700818:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776528.700819:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776528.700823:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776528.700825:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776528.700828:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776528.700830:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776528.700833:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776528.700836:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710421376, portal 10 00000100:00000001:1.0:1775776528.700838:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776528.700839:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004928 : -123823860546688 : ffff8f6202c70380) 00000100:00000040:1.0:1775776528.700844:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70380 x1862022710421376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776553 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.700850:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776528.700852:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776528.700855:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776528.700857:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710421376, offset 0 00000400:00000200:1.0:1775776528.700861:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776528.700864:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776528.700867:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776528.700870:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776528.700873:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776528.700875:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776528.700877:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776528.700880:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453844:453844:256:4294967295] 192.168.201.115@tcp LPNI seq info [453769:453769:8:4294967295] 00000400:00000200:1.0:1775776528.700884:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776528.700890:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776528.700895:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776528.700898:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776528.700901:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776528.700904:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775776528.700908:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776528.700910:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775776528.700913:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776528.700916:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776528.700918:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776528.700921:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776528.700924:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776528.700926:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776528.700928:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776528.700941:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70380 x1862022710421376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776553 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.700949:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776528.700951:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70380 x1862022710421376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776553 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.700957:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776528.700959:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776528.700962:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776528.700964:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776528.700968:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776528.700970:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308083724 00000100:00000001:1.0:1775776528.700972:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308087559 00000100:00000001:1.0:1775776528.700975:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308083724 00000100:00000001:1.0:1775776528.700977:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308087559 00000100:00000001:1.0:1775776528.700979:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776528.700980:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776528.700982:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776528.700985:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776528/real 1775776528] req@ffff8f6202c70380 x1862022710421376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776528.700993:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776528.700994:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776528.700996:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776528.700998:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776528.700999:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776528.701000:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776528.701003:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70380 x1862022710421376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776528.701011:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70380 x1862022710421376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.701017:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776528.701018:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776528.701019:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776528.701021:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776528.701024:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70380 x1862022710421376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.701029:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776528.701033:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70380 x1862022710421376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.701039:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776528.701041:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776528.701042:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776528.701043:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776528.701045:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776528.701046:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776528.701049:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776528.701052:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776528.701054:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776528.701057:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70380 x1862022710421376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776528.701064:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710421376:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776528.701067:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776528.701069:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776528.701071:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73480 x1862022710421248/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.701077:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776528.701079:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776528.701081:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776528.701083:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776528.701084:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776528.701085:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776528.701088:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776528.701089:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37c00. 02000000:00000001:1.0:1775776528.701092:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776528.701093:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776528.701094:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73480. 00000100:00000001:1.0:1775776528.701097:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776528.701098:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776528.701099:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776528.701102:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70380 x1862022710421376/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776553 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776528.701109:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776528.701110:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776528.701112:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776528.701114:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776528.701115:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776528.701116:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776528.701118:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776528.701119:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37800. 02000000:00000001:1.0:1775776528.701121:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776528.701123:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776528.701124:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70380. 00000100:00000001:1.0:1775776528.701126:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776528.701127:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776528.701129:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776528.701132:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776528.701153:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776528.701155:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776528.701157:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776528.827255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776528.827262:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776528.827269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776528.827270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776528.827275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776528.827276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776528.827341:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776528.827346:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776528.827349:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776529.723307:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776529.723317:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776529.723319:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776529.723328:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776529.723333:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776529.723338:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776529.723341:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776529.723345:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776529.723348:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776529.852188:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776529.852196:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776529.852203:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776529.852204:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776529.852207:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776529.852209:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776529.852271:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776529.852275:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776529.852279:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776530.747277:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776530.747286:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776530.747288:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776530.747295:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776530.747299:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776530.747303:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776530.747305:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776530.747308:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776530.747310:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776530.875279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776530.875286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776530.875301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776530.875303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776530.875307:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776530.875308:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776530.875465:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776530.875469:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776530.875473:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776531.771337:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776531.771345:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776531.771347:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776531.771353:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776531.771358:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776531.771362:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776531.771365:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776531.771368:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776531.771370:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776531.899286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776531.899292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776531.899299:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776531.899302:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776531.899304:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776531.899308:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776531.899310:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776531.899313:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776531.899314:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776531.899317:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776531.899320:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776531.899322:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776531.899323:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776531.899324:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776531.899326:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776531.899328:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776531.899329:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776531.899331:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776531.899382:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776531.899385:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776531.899388:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776532.603228:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308087559, at 4308087616 00000100:00000010:1.0:1775776532.603242:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2de0. 00000100:00000040:0.0:1775776532.603244:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308087559, at 4308087616 00000100:00000010:0.0:1775776532.603253:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2660. 00000100:00000001:1.0:1775776532.795313:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776532.795320:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776532.795322:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776532.795327:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776532.795330:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776532.795334:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776532.795341:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776532.795344:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776532.795347:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776532.923271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776532.923276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776532.923283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776532.923284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776532.923288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776532.923289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776532.923358:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776532.923362:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776532.923366:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776533.819247:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1775776533.819252:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776533.819254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776533.819256:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776533.819263:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:2.0:1775776533.819264:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776533.819267:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776533.819268:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776533.819271:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776533.819274:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1775776533.819275:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776533.819276:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776533.819280:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776533.819281:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776533.819289:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13416 00000400:00000200:2.0:1775776533.819299:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776533.819305:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776533.819308:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776533.819310:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776533.819312:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776533.819314:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776533.819316:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776533.819318:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776533.819319:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776533.819322:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776533.819324:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776533.819325:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776533.819327:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776533.819329:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776533.819330:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776533.819331:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776533.819332:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776533.819335:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776533.819337:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776533.819339:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776533.819342:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776533.819344:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776533.819346:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776533.819348:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776533.819350:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776533.819359:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775776533.819363:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776533.819368:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776533.819370:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776533.819372:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776533.819375:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776533.819379:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776533.819389:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775776533.819396:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776533.819407:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710421504/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776533.819418:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776533.819422:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776533.819426:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710421504/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776533.819442:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776533.819444:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776533.819447:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776533.819448:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776533.819449:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776533.819451:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776533.819454:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13416 00000400:00000200:2.0:1775776533.819457:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776533.819460:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:1.0:1775776533.819461:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1775776533.819461:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776533.819465:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1775776533.819465:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776533.819468:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1775776533.819468:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776533.819471:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776533.819473:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:1.0:1775776533.819475:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344a680 x1862022710421504/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775776533.819475:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776533.819476:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776533.819478:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776533.819481:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776533.819484:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776533.819487:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776533.819493:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776533.819494:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776533.819495:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776533.819497:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776533.819498:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776533.819502:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776533.819503:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:1.0:1775776533.819504:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1775776533.819505:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000001:1.0:1775776533.819508:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776533.819508:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776533.819509:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776533.819513:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776533.819515:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1775776533.819518:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710421504:192.168.201.115@tcp:250:kworker.0 02000000:00000001:2.0:1775776533.819523:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776533.819524:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1775776533.819525:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776533.819530:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:2.0:1775776533.819530:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776533.819532:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776533.819535:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776533.819535:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776533.819540:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:2.0:1775776533.819542:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775776533.819545:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776533.819546:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000010:1.0:1775776533.819547:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:2.0:1775776533.819547:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776533.819551:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776533.819552:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776533.819554:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776533.819556:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776533.819560:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000010:2.0:1775776533.819560:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775776533.819564:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776533.819567:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00100000:2.0:1775776533.819568:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710421632/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776533.819574:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776533.819576:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776533.819577:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000040:2.0:1775776533.819579:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710421632/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776533.819586:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710421504, portal 25 00000100:00000001:1.0:1775776533.819590:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776533.819592:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662545024 : -123823047006592 : ffff8f623344a680) 00000100:00000001:2.0:1775776533.819595:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776533.819600:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344a680 x1862022710421504/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776558 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775776533.819600:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776533.819608:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776533.819612:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776533.819614:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776533.819617:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1775776533.819619:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000040:1.0:1775776533.819622:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00080000:2.0:1775776533.819625:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13416 00000400:00000010:1.0:1775776533.819627:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000400:00000200:2.0:1775776533.819632:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000200:1.0:1775776533.819634:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710421504, offset 0 00000400:00000200:2.0:1775776533.819636:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776533.819640:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1775776533.819644:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000040:2.0:1775776533.819645:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776533.819648:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776533.819650:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776533.819651:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776533.819653:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000200:1.0:1775776533.819657:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000040:2.0:1775776533.819658:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776533.819660:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776533.819662:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776533.819663:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776533.819666:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776533.819667:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000040:2.0:1775776533.819672:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776533.819674:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776533.819676:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1775776533.819677:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776533.819680:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776533.819682:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776533.819686:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00080000:2.0:1775776533.819686:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776533.819688:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776533.819690:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776533.819693:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776533.819697:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 02000000:00000001:2.0:1775776533.819698:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776533.819700:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776533.819703:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775776533.819704:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453845:453845:256:4294967295] 192.168.201.115@tcp LPNI seq info [453770:453770:8:4294967295] 00000008:00100000:2.0:1775776533.819705:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776533.819710:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776533.819712:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776533.819715:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000010:2.0:1775776533.819716:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000020:00000040:2.0:1775776533.819719:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776533.819722:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776533.819723:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776533.819727:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1775776533.819730:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 02000000:00000001:2.0:1775776533.819730:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776533.819734:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776533.819738:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 02000000:00000010:2.0:1775776533.819739:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000800:00000010:1.0:1775776533.819744:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000100:00000001:2.0:1775776533.819744:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776533.819752:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776533.819756:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000100:00100000:2.0:1775776533.819759:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710421760/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1775776533.819763:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776533.819768:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000100:00000001:2.0:1775776533.819773:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000100:1.0:1775776533.819776:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000001:2.0:1775776533.819777:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775776533.819780:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000100:00000040:2.0:1775776533.819782:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710421760/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776533.819787:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1775776533.819788:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776533.819791:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776533.819793:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000040:2.0:1775776533.819797:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1775776533.819800:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000100:00000040:2.0:1775776533.819800:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:1.0:1775776533.819803:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000040:2.0:1775776533.819804:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776533.819811:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:2.0:1775776533.819811:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13426) 00000100:00000200:1.0:1775776533.819815:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344a680 x1862022710421504/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776558 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.819822:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776533.819824:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344a680 x1862022710421504/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776558 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.819829:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776533.819830:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776533.819835:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776533.819838:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776533.819851:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308092678 00000100:00000010:1.0:1775776533.819857:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2de0. 00000100:00000001:1.0:1775776533.819863:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776533.819868:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308092678 00000100:00000010:1.0:1775776533.819870:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2300. 00000100:00000001:1.0:1775776533.819872:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776533.819874:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776533.819876:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776533.819878:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776533.819881:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776533/real 1775776533] req@ffff8f623344a680 x1862022710421504/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776533.819890:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776533.819892:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776533.819894:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776533.819897:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776533.819898:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776533.819901:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776533.819904:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344a680 x1862022710421504/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776533.819913:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344a680 x1862022710421504/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.819919:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776533.819921:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776533.819923:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776533.819926:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776533.819929:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344a680 x1862022710421504/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.819936:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776533.819940:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344a680 x1862022710421504/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.819949:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776533.819951:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776533.819952:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776533.819953:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776533.819954:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776533.819956:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776533.819958:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776533.819961:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776533.819962:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776533.819964:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344a680 x1862022710421504/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776533.819969:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710421504:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776533.819971:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776533.819973:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776533.819975:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344a680 x1862022710421504/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.819979:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776533.819981:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776533.819983:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776533.819985:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776533.819986:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776533.819987:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776533.819989:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776533.819991:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4cc00. 02000000:00000001:1.0:1775776533.819995:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776533.819996:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776533.819997:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344a680. 00000100:00000001:1.0:1775776533.820000:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776533.820001:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776533.820002:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776533.820010:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776533.820010:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776533.820012:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776533.820013:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776533.820014:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776533.820016:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449f80 x1862022710421632/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.820020:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776533.820021:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776533.820022:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776533.820023:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776533.820024:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776533.820026:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710421632:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776533.820028:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776533.820029:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776533.820029:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776533.820030:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776533.820032:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776533.820033:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776533.820042:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776533.820044:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776533.820047:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776533.820051:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710421632, portal 10 00000100:00000001:1.0:1775776533.820052:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776533.820053:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662543232 : -123823047008384 : ffff8f6233449f80) 00000100:00000040:1.0:1775776533.820058:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449f80 x1862022710421632/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776558 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.820062:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776533.820064:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776533.820065:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776533.820066:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710421632, offset 0 00000400:00000200:1.0:1775776533.820069:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776533.820071:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776533.820074:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776533.820076:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776533.820077:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776533.820079:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776533.820080:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776533.820082:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453846:453846:256:4294967295] 192.168.201.115@tcp LPNI seq info [453771:453771:8:4294967295] 00000400:00000200:1.0:1775776533.820085:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776533.820089:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776533.820091:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776533.820093:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776533.820095:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776533.820097:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775776533.820100:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776533.820101:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1775776533.820104:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776533.820105:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776533.820107:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776533.820109:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776533.820111:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776533.820112:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776533.820114:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776533.820116:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449f80 x1862022710421632/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776558 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.820121:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776533.820122:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449f80 x1862022710421632/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776558 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.820126:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776533.820127:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776533.820129:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776533.820131:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776533.820147:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776533.820149:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308088844 00000100:00000001:1.0:1775776533.820151:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308092678 00000100:00000001:1.0:1775776533.820153:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308088844 00000100:00000001:1.0:1775776533.820154:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308092678 00000100:00000001:1.0:1775776533.820156:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776533.820157:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776533.820158:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776533.820160:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776533/real 1775776533] req@ffff8f6233449f80 x1862022710421632/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776533.820164:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776533.820165:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776533.820166:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776533.820168:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776533.820168:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776533.820170:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776533.820171:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449f80 x1862022710421632/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776533.820176:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449f80 x1862022710421632/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.820179:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776533.820180:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776533.820181:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776533.820182:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776533.820183:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449f80 x1862022710421632/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.820187:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776533.820189:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449f80 x1862022710421632/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.820192:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776533.820194:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776533.820195:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776533.820196:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776533.820197:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776533.820201:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776533.820203:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776533.820205:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776533.820206:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776533.820208:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449f80 x1862022710421632/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776533.820213:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710421632:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776533.820215:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776533.820216:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233448380 x1862022710421760/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.820219:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776533.820220:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776533.820221:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776533.820222:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776533.820223:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776533.820225:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233448380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710421760:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776533.820227:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776533.820228:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776533.820229:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776533.820230:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776533.820232:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776533.820233:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776533.820235:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776533.820236:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776533.820238:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776533.820239:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710421760, portal 10 00000100:00000001:1.0:1775776533.820241:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776533.820242:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662536064 : -123823047015552 : ffff8f6233448380) 00000100:00000040:1.0:1775776533.820246:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233448380 x1862022710421760/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776558 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.820250:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776533.820251:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776533.820252:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776533.820254:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710421760, offset 0 00000400:00000200:1.0:1775776533.820256:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776533.820258:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776533.820260:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776533.820261:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776533.820263:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776533.820264:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776533.820266:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776533.820268:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453847:453847:256:4294967295] 192.168.201.115@tcp LPNI seq info [453772:453772:8:4294967295] 00000400:00000200:1.0:1775776533.820270:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776533.820274:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776533.820277:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776533.820278:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776533.820281:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776533.820282:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1775776533.820285:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776533.820286:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1775776533.820289:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776533.820290:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776533.820292:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776533.820294:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776533.820295:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776533.820296:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776533.820298:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776533.820300:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233448380 x1862022710421760/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776558 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.820308:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776533.820310:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448380 x1862022710421760/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776558 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.820313:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776533.820314:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776533.820316:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776533.820317:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776533.820319:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776533.820321:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308088844 00000100:00000001:1.0:1775776533.820323:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308092678 00000100:00000001:1.0:1775776533.820325:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308088844 00000100:00000001:1.0:1775776533.820327:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308092678 00000100:00000001:1.0:1775776533.820329:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776533.820331:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776533.820333:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776533.820336:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776533/real 1775776533] req@ffff8f6233448380 x1862022710421760/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776533.820344:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776533.820345:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776533.820346:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776533.820349:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776533.820351:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776533.820353:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776533.820356:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233448380 x1862022710421760/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776533.820364:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233448380 x1862022710421760/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.820371:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776533.820373:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776533.820374:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776533.820376:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776533.820378:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233448380 x1862022710421760/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.820382:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776533.820385:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233448380 x1862022710421760/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.820389:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776533.820390:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776533.820391:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776533.820391:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776533.820392:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776533.820393:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776533.820395:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776533.820398:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776533.820400:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776533.820403:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233448380 x1862022710421760/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776533.820409:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233448380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710421760:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776533.820413:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776533.820415:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776533.820417:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449f80 x1862022710421632/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.820423:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776533.820425:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776533.820427:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776533.820429:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776533.820430:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776533.820432:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776533.820434:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776533.820436:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4d400. 02000000:00000001:1.0:1775776533.820438:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776533.820439:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776533.820440:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449f80. 00000100:00000001:1.0:1775776533.820443:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776533.820444:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776533.820446:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776533.820449:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233448380 x1862022710421760/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776558 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776533.820455:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776533.820457:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776533.820458:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776533.820460:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776533.820461:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776533.820462:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776533.820464:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776533.820465:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4d800. 02000000:00000001:1.0:1775776533.820468:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776533.820469:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776533.820470:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233448380. 00000100:00000001:1.0:1775776533.820472:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776533.820473:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776533.820475:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776533.820478:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776533.820480:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776533.820481:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776533.820483:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776533.947291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776533.947299:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776533.947306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776533.947307:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776533.947311:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776533.947312:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776533.947373:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776533.947378:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776533.947382:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776534.843243:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776534.843250:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776534.843252:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776534.843258:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776534.843261:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776534.843266:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776534.843269:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776534.843271:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776534.843274:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776534.971321:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776534.971327:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776534.971332:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776534.971333:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776534.971336:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776534.971337:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776534.971424:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776534.971430:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776534.971433:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776535.867251:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776535.867271:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776535.867273:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776535.867279:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776535.867283:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776535.867286:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776535.867289:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776535.867291:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776535.867294:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776535.995261:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776535.995266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776535.995271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776535.995272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776535.995276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776535.995277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776535.995346:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776535.995350:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776535.995353:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776536.891238:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776536.891244:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776536.891245:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776536.891250:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776536.891253:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776536.891255:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776536.891258:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776536.891259:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776536.891261:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776537.019232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776537.019237:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776537.019239:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776537.019241:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776537.019245:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776537.019247:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776537.019248:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776537.019249:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776537.019251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776537.019252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776537.019255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776537.019256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776537.019319:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776537.019323:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776537.019326:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776537.723294:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308092678, at 4308092736 00000100:00000040:0.0:1775776537.723317:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308092678, at 4308092736 00000100:00000010:1.0:1775776537.723317:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2300. 00000100:00000010:0.0:1775776537.723328:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2de0. 00000100:00000001:1.0:1775776537.915275:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776537.915284:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776537.915286:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776537.915294:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776537.915299:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776537.915304:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776537.915307:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776537.915310:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776537.915313:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776538.043269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776538.043274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776538.043279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776538.043280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776538.043283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776538.043284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776538.043349:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776538.043354:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776538.043358:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776538.939228:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776538.939234:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776538.939236:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776538.939240:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776538.939243:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776538.939246:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776538.939248:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775776538.939250:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776538.939251:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776538.939254:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776538.939258:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776538.939262:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776538.939265:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776538.939269:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776538.939276:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13421 00000400:00000200:3.0:1775776538.939286:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776538.939292:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776538.939295:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776538.939298:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776538.939302:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776538.939304:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776538.939306:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776538.939310:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776538.939311:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776538.939314:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776538.939317:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776538.939319:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776538.939321:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776538.939324:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776538.939326:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776538.939327:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776538.939329:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776538.939332:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776538.939335:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776538.939337:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776538.939341:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776538.939344:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776538.939347:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776538.939351:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776538.939353:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776538.939362:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775776538.939365:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776538.939369:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776538.939370:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776538.939372:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776538.939375:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776538.939378:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776538.939384:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:3.0:1775776538.939390:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776538.939396:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710421888/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776538.939403:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776538.939406:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776538.939409:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710421888/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776538.939423:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776538.939426:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776538.939428:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776538.939429:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776538.939430:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776538.939432:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776538.939433:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13421 00000400:00000200:3.0:1775776538.939436:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776538.939438:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776538.939440:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776538.939442:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776538.939444:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776538.939445:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776538.939447:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776538.939449:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775776538.939450:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1775776538.939450:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:1.0:1775776538.939453:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1775776538.939453:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776538.939455:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:1.0:1775776538.939456:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1775776538.939459:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:1.0:1775776538.939462:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71500 x1862022710421888/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775776538.939463:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776538.939471:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776538.939473:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776538.939475:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776538.939475:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775776538.939479:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1775776538.939479:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776538.939483:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776538.939484:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1775776538.939487:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:1.0:1775776538.939488:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776538.939492:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00100000:1.0:1775776538.939495:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710421888:192.168.201.115@tcp:250:kworker.0 00000100:00000001:3.0:1775776538.939497:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776538.939501:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776538.939503:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776538.939505:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776538.939508:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776538.939508:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776538.939511:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776538.939513:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775776538.939515:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775776538.939516:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:3.0:1775776538.939518:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776538.939520:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776538.939523:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 00000100:00000010:3.0:1775776538.939523:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 02000000:00000001:1.0:1775776538.939527:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776538.939527:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776538.939529:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776538.939531:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776538.939532:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776538.939533:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776538.939534:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776538.939537:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000010:3.0:1775776538.939539:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000001:3.0:1775776538.939542:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776538.939544:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00100000:3.0:1775776538.939547:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710422016/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1775776538.939551:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000001:3.0:1775776538.939553:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776538.939557:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776538.939560:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710421888, portal 25 00000100:00000040:3.0:1775776538.939561:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710422016/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.939564:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776538.939567:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849009408 : -123823860542208 : ffff8f6202c71500) 00000100:00000040:1.0:1775776538.939577:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71500 x1862022710421888/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776563 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776538.939577:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776538.939586:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776538.939589:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00080000:3.0:1775776538.939592:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:1.0:1775776538.939593:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776538.939597:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710421888, offset 0 00000100:00080000:3.0:1775776538.939598:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776538.939601:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776538.939603:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775776538.939604:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776538.939609:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776538.939612:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13421 00000400:00000200:1.0:1775776538.939615:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776538.939618:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776538.939622:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:3.0:1775776538.939622:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776538.939626:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1775776538.939629:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1775776538.939630:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776538.939633:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776538.939634:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776538.939637:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:3.0:1775776538.939638:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776538.939641:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776538.939642:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776538.939643:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:1.0:1775776538.939646:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000020:00000040:3.0:1775776538.939646:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776538.939647:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776538.939650:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776538.939652:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000040:3.0:1775776538.939654:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776538.939656:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776538.939659:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776538.939662:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453848:453848:256:4294967295] 192.168.201.115@tcp LPNI seq info [453773:453773:8:4294967295] 00000100:00000001:3.0:1775776538.939662:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776538.939665:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776538.939669:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000400:00000200:1.0:1775776538.939672:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776538.939672:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776538.939673:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776538.939677:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776538.939679:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776538.939680:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776538.939682:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776538.939685:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776538.939688:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776538.939690:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000010:3.0:1775776538.939692:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775776538.939695:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776538.939698:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776538.939700:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776538.939701:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776538.939702:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776538.939705:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776538.939706:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775776538.939708:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 02000000:00000010:3.0:1775776538.939709:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775776538.939714:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776538.939718:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710422144/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1775776538.939719:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776538.939725:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000100:00000001:3.0:1775776538.939727:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776538.939729:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776538.939732:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710422144/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1775776538.939736:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776538.939742:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000100:00000001:3.0:1775776538.939743:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776538.939746:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776538.939761:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000100:1.0:1775776538.939763:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776538.939764:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:1.0:1775776538.939768:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000100:00000040:3.0:1775776538.939768:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776538.939773:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13431) 00000400:00000200:1.0:1775776538.939783:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776538.939789:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776538.939794:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776538.939797:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776538.939802:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776538.939806:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71500 x1862022710421888/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776563 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.939814:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776538.939817:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71500 x1862022710421888/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776563 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.939824:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776538.939826:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776538.939831:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776538.939835:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776538.939837:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308097798 00000100:00000010:1.0:1775776538.939843:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e25a0. 00000100:00000001:1.0:1775776538.939849:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776538.939860:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308097798 00000100:00000010:1.0:1775776538.939863:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2120. 00000100:00000001:1.0:1775776538.939866:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776538.939869:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776538.939871:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776538.939873:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776538.939877:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776538/real 1775776538] req@ffff8f6202c71500 x1862022710421888/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776538.939885:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776538.939887:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776538.939889:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776538.939892:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776538.939894:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776538.939897:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776538.939900:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71500 x1862022710421888/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776538.939909:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71500 x1862022710421888/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.939915:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776538.939917:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776538.939919:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776538.939921:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776538.939924:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71500 x1862022710421888/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.939931:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776538.939934:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71500 x1862022710421888/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.939942:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776538.939944:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776538.939946:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776538.939947:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776538.939950:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776538.939952:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776538.939954:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776538.939957:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776538.939960:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776538.939963:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71500 x1862022710421888/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776538.939971:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710421888:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776538.939975:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776538.939976:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776538.939978:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71500 x1862022710421888/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.939984:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776538.939986:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776538.939989:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776538.939991:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776538.939993:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776538.939994:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776538.939996:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776538.939998:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35400. 02000000:00000001:1.0:1775776538.940001:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776538.940003:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776538.940005:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71500. 00000100:00000001:1.0:1775776538.940007:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776538.940009:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776538.940011:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776538.940017:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776538.940019:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776538.940020:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776538.940022:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776538.940023:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776538.940026:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70a80 x1862022710422016/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.940033:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776538.940034:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776538.940036:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776538.940038:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776538.940040:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776538.940043:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710422016:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776538.940046:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776538.940048:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776538.940049:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776538.940051:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776538.940054:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776538.940055:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776538.940059:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776538.940062:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776538.940065:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776538.940068:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710422016, portal 10 00000100:00000001:1.0:1775776538.940070:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776538.940071:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849006720 : -123823860544896 : ffff8f6202c70a80) 00000100:00000040:1.0:1775776538.940077:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70a80 x1862022710422016/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776563 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.940083:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776538.940085:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776538.940087:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776538.940089:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710422016, offset 0 00000400:00000200:1.0:1775776538.940093:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776538.940097:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776538.940100:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776538.940103:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776538.940105:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776538.940108:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776538.940110:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776538.940113:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453849:453849:256:4294967295] 192.168.201.115@tcp LPNI seq info [453774:453774:8:4294967295] 00000400:00000200:1.0:1775776538.940118:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776538.940124:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776538.940128:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776538.940132:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776538.940153:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776538.940155:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775776538.940160:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776538.940162:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1775776538.940166:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776538.940168:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776538.940171:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776538.940174:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776538.940177:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776538.940179:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776538.940182:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776538.940185:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70a80 x1862022710422016/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776563 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.940193:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776538.940195:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70a80 x1862022710422016/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776563 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.940200:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776538.940202:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776538.940204:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776538.940208:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776538.940211:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776538.940213:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308093964 00000100:00000001:1.0:1775776538.940216:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308097798 00000100:00000001:1.0:1775776538.940219:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308093964 00000100:00000001:1.0:1775776538.940221:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308097798 00000100:00000001:1.0:1775776538.940224:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776538.940226:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776538.940227:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776538.940230:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776538/real 1775776538] req@ffff8f6202c70a80 x1862022710422016/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776538.940237:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776538.940239:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776538.940240:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776538.940242:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776538.940243:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776538.940245:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776538.940248:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70a80 x1862022710422016/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776538.940256:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70a80 x1862022710422016/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.940262:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776538.940264:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776538.940266:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776538.940267:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776538.940271:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70a80 x1862022710422016/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.940277:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776538.940281:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70a80 x1862022710422016/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.940288:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776538.940291:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776538.940293:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776538.940294:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776538.940296:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776538.940300:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776538.940303:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776538.940306:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776538.940308:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776538.940312:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70a80 x1862022710422016/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776538.940320:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710422016:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776538.940324:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776538.940327:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70700 x1862022710422144/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.940334:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776538.940336:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776538.940338:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776538.940339:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776538.940342:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776538.940344:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710422144:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776538.940348:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776538.940349:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776538.940351:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776538.940352:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776538.940355:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776538.940357:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776538.940361:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776538.940363:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776538.940366:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776538.940369:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710422144, portal 10 00000100:00000001:1.0:1775776538.940370:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776538.940373:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849005824 : -123823860545792 : ffff8f6202c70700) 00000100:00000040:1.0:1775776538.940377:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70700 x1862022710422144/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776563 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.940384:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776538.940386:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776538.940388:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776538.940391:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710422144, offset 0 00000400:00000200:1.0:1775776538.940394:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776538.940398:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776538.940401:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776538.940404:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776538.940406:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776538.940409:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776538.940411:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776538.940414:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453850:453850:256:4294967295] 192.168.201.115@tcp LPNI seq info [453775:453775:8:4294967295] 00000400:00000200:1.0:1775776538.940418:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776538.940424:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776538.940428:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776538.940431:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776538.940435:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776538.940438:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775776538.940442:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776538.940445:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1775776538.940448:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776538.940450:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776538.940453:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776538.940457:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776538.940459:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776538.940461:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776538.940463:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776538.940467:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70700 x1862022710422144/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776563 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.940479:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776538.940482:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70700 x1862022710422144/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776563 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.940489:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776538.940490:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776538.940493:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776538.940496:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776538.940499:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776538.940501:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308093964 00000100:00000001:1.0:1775776538.940503:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308097798 00000100:00000001:1.0:1775776538.940505:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308093964 00000100:00000001:1.0:1775776538.940507:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308097798 00000100:00000001:1.0:1775776538.940509:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776538.940511:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776538.940513:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776538.940516:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776538/real 1775776538] req@ffff8f6202c70700 x1862022710422144/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776538.940524:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776538.940525:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776538.940526:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776538.940528:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776538.940530:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776538.940532:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776538.940535:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70700 x1862022710422144/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776538.940543:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70700 x1862022710422144/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.940549:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776538.940551:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776538.940553:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776538.940554:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776538.940557:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70700 x1862022710422144/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.940563:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776538.940567:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70700 x1862022710422144/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.940574:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776538.940576:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776538.940577:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776538.940579:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776538.940580:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776538.940582:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776538.940585:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776538.940588:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776538.940590:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776538.940593:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70700 x1862022710422144/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776538.940601:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710422144:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776538.940605:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776538.940606:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776538.940609:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70a80 x1862022710422016/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.940615:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776538.940616:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776538.940618:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776538.940621:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776538.940622:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776538.940623:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776538.940625:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776538.940627:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35800. 02000000:00000001:1.0:1775776538.940629:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776538.940630:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776538.940632:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70a80. 00000100:00000001:1.0:1775776538.940634:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776538.940635:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776538.940637:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776538.940639:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70700 x1862022710422144/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776563 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776538.940645:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776538.940646:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776538.940648:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776538.940650:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776538.940651:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776538.940652:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776538.940654:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776538.940656:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37400. 02000000:00000001:1.0:1775776538.940658:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776538.940659:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776538.940660:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70700. 00000100:00000001:1.0:1775776538.940661:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776538.940663:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776538.940665:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776538.940668:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776538.940669:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776538.940671:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776538.940672:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776539.067238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776539.067244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776539.067249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776539.067250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776539.067254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776539.067255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776539.067315:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776539.067318:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776539.067320:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776539.963302:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776539.963312:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776539.963314:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776539.963322:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776539.963327:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776539.963332:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776539.963336:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776539.963339:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776539.963342:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776540.091279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776540.091287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776540.091294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776540.091294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776540.091297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776540.091298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776540.091367:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776540.091372:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776540.091375:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776540.987202:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776540.987207:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776540.987209:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776540.987212:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776540.987215:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776540.987218:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776540.987220:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776540.987221:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776540.987223:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776541.115238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776541.115245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776541.115250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776541.115251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776541.115255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776541.115257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776541.115318:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776541.115322:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776541.115325:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776542.011249:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776542.011254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776542.011256:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776542.011260:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776542.011262:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776542.011264:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776542.011266:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776542.011268:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776542.011270:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776542.139245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776542.139250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776542.139254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776542.139256:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776542.139257:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776542.139260:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776542.139262:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776542.139264:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776542.139265:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776542.139273:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776542.139276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776542.139277:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776542.139278:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776542.139279:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776542.139282:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776542.139283:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776542.139284:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776542.139285:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776542.139354:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776542.139358:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776542.139361:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776542.843233:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308097798, at 4308097856 00000100:00000040:1.0:1775776542.843233:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308097798, at 4308097856 00000100:00000010:0.0:1775776542.843247:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e25a0. 00000100:00000010:1.0:1775776542.843251:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2120. 00000100:00000001:1.0:1775776543.035247:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776543.035253:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776543.035255:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776543.035260:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776543.035264:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776543.035267:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776543.035270:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776543.035272:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776543.035274:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776543.163242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776543.163247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776543.163250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776543.163251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776543.163254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776543.163255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776543.163306:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776543.163310:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776543.163312:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776544.059231:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776544.059232:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776544.059239:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1775776544.059240:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776544.059242:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776544.059244:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1775776544.059249:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:3.0:1775776544.059249:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776544.059254:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776544.059255:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776544.059259:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776544.059262:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776544.059265:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776544.059268:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776544.059270:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13426 00000400:00000200:3.0:1775776544.059280:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776544.059286:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776544.059289:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776544.059292:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776544.059294:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776544.059296:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776544.059298:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776544.059300:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776544.059302:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776544.059304:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776544.059306:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776544.059307:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776544.059309:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776544.059310:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776544.059312:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776544.059313:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776544.059314:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776544.059316:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776544.059319:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776544.059321:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776544.059324:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776544.059328:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776544.059331:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776544.059333:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776544.059335:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776544.059349:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775776544.059353:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776544.059357:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776544.059359:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776544.059361:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776544.059363:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776544.059366:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776544.059372:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000001:3.0:1775776544.059376:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776544.059384:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710422272/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776544.059391:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776544.059394:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776544.059397:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710422272/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776544.059418:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776544.059421:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776544.059423:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776544.059424:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776544.059425:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776544.059427:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776544.059429:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13426 00000400:00000200:3.0:1775776544.059433:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776544.059435:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1775776544.059435:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:1.0:1775776544.059439:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776544.059439:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776544.059442:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776544.059443:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776544.059445:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776544.059449:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72a00 x1862022710422272/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776544.059450:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776544.059454:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776544.059458:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776544.059460:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776544.059462:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775776544.059465:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1775776544.059465:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:1.0:1775776544.059468:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775776544.059468:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776544.059472:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776544.059474:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1775776544.059474:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776544.059477:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775776544.059478:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1775776544.059479:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776544.059482:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776544.059484:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776544.059484:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00100000:1.0:1775776544.059492:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710422272:192.168.201.115@tcp:250:kworker.0 00000100:00080000:3.0:1775776544.059492:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776544.059493:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776544.059496:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776544.059500:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776544.059501:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776544.059503:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776544.059505:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1775776544.059505:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776544.059509:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775776544.059509:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776544.059513:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776544.059514:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775776544.059514:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776544.059519:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 02000000:00000010:1.0:1775776544.059521:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4fc00. 00000020:00000040:3.0:1775776544.059521:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776544.059525:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776544.059532:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776544.059534:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000010:1.0:1775776544.059538:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 02000000:00000001:3.0:1775776544.059539:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776544.059543:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776544.059544:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1775776544.059545:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 02000000:00000001:3.0:1775776544.059546:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776544.059552:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000100:00000200:1.0:1775776544.059553:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710422272, portal 25 00000100:00000001:3.0:1775776544.059556:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776544.059557:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776544.059560:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849014784 : -123823860536832 : ffff8f6202c72a00) 00000100:00100000:3.0:1775776544.059564:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710422400/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1775776544.059566:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72a00 x1862022710422272/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.059593:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775776544.059594:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1775776544.059598:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776544.059598:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776544.059602:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000040:3.0:1775776544.059603:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710422400/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776544.059604:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710422272, offset 0 00000400:00000200:1.0:1775776544.059613:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776544.059614:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776544.059617:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776544.059619:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776544.059620:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776544.059622:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775776544.059623:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776544.059626:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776544.059629:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13426 00000400:00000200:1.0:1775776544.059633:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:3.0:1775776544.059634:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776544.059639:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000400:00000200:1.0:1775776544.059642:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:3.0:1775776544.059642:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776544.059646:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776544.059648:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:3.0:1775776544.059648:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776544.059652:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776544.059654:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776544.059656:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776544.059659:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776544.059672:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453851:453851:256:4294967295] 192.168.201.115@tcp LPNI seq info [453776:453776:8:4294967295] 00000020:00000001:3.0:1775776544.059676:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776544.059679:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000400:00000200:1.0:1775776544.059680:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000001:3.0:1775776544.059681:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776544.059683:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776544.059686:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776544.059687:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776544.059691:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776544.059694:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776544.059698:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776544.059699:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:3.0:1775776544.059700:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776544.059704:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776544.059708:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000800:00000200:1.0:1775776544.059711:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776544.059712:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776544.059716:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776544.059718:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:1.0:1775776544.059719:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 02000000:00000001:3.0:1775776544.059720:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776544.059723:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776544.059725:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776544.059728:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776544.059730:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775776544.059730:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776544.059734:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000800:00000200:1.0:1775776544.059736:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000020:00000040:3.0:1775776544.059737:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776544.059742:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776544.059744:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776544.059747:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.1:1775776544.059747:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:3.0:1775776544.059749:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776544.059752:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776544.059753:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 02000000:00000010:3.0:1775776544.059755:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000001:3.0:1775776544.059758:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1775776544.059760:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776544.059763:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000100:00100000:3.0:1775776544.059764:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710422528/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776544.059771:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1775776544.059771:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776544.059774:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776544.059779:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710422528/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776544.059781:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1775776544.059788:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776544.059789:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000100:00000040:3.0:1775776544.059790:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:1.0:1775776544.059794:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000040:3.0:1775776544.059794:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776544.059798:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776544.059801:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776544.059802:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1775776544.059807:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13436) 00000100:00000200:1.0:1775776544.059809:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72a00 x1862022710422272/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.059818:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776544.059822:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72a00 x1862022710422272/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776569 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.059829:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776544.059830:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776544.059836:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776544.059840:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776544.059842:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308102918 00000100:00000010:1.0:1775776544.059846:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2120. 00000100:00000001:1.0:1775776544.059851:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776544.059861:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308102918 00000100:00000010:1.0:1775776544.059863:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2f60. 00000100:00000001:1.0:1775776544.059865:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776544.059868:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776544.059869:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776544.059871:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776544.059874:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776544/real 1775776544] req@ffff8f6202c72a00 x1862022710422272/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776544.059880:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776544.059881:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776544.059882:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776544.059884:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776544.059885:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776544.059887:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776544.059890:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72a00 x1862022710422272/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776544.059895:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72a00 x1862022710422272/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.059899:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776544.059901:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776544.059902:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776544.059904:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776544.059906:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72a00 x1862022710422272/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.059909:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776544.059912:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72a00 x1862022710422272/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.059916:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776544.059918:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776544.059919:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776544.059919:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776544.059921:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776544.059922:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776544.059924:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776544.059926:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776544.059927:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776544.059929:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72a00 x1862022710422272/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776544.059933:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710422272:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776544.059936:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776544.059937:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776544.059938:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72a00 x1862022710422272/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.059942:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776544.059944:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776544.059946:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776544.059947:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776544.059949:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776544.059950:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776544.059951:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776544.059953:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37000. 02000000:00000001:1.0:1775776544.059956:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776544.059957:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776544.059958:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72a00. 00000100:00000001:1.0:1775776544.059960:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776544.059961:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776544.059962:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776544.059967:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776544.059968:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776544.059969:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776544.059970:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776544.059971:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776544.059973:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72300 x1862022710422400/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.059977:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776544.059978:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776544.059979:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776544.059980:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776544.059981:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776544.059983:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710422400:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776544.059984:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776544.059986:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776544.059986:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776544.059987:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776544.059989:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776544.059990:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776544.059992:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776544.059994:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776544.059996:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776544.059998:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710422400, portal 10 00000100:00000001:1.0:1775776544.059999:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776544.060000:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849012992 : -123823860538624 : ffff8f6202c72300) 00000100:00000040:1.0:1775776544.060004:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72300 x1862022710422400/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.060008:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776544.060009:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776544.060010:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776544.060011:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710422400, offset 0 00000400:00000200:1.0:1775776544.060014:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776544.060016:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776544.060025:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776544.060027:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776544.060028:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776544.060030:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776544.060031:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776544.060033:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453852:453852:256:4294967295] 192.168.201.115@tcp LPNI seq info [453777:453777:8:4294967295] 00000400:00000200:1.0:1775776544.060036:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776544.060040:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776544.060042:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776544.060044:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776544.060046:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776544.060048:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775776544.060051:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776544.060052:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1775776544.060054:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776544.060055:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776544.060057:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776544.060059:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776544.060060:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776544.060062:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776544.060063:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776544.060065:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72300 x1862022710422400/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.060069:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776544.060071:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72300 x1862022710422400/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776569 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.060075:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776544.060076:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776544.060077:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776544.060080:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776544.060081:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776544.060082:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308099083 00000100:00000001:1.0:1775776544.060084:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308102918 00000100:00000001:1.0:1775776544.060086:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308099083 00000100:00000001:1.0:1775776544.060087:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308102918 00000100:00000001:1.0:1775776544.060088:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776544.060089:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776544.060090:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776544.060092:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776544/real 1775776544] req@ffff8f6202c72300 x1862022710422400/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776544.060096:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776544.060097:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776544.060098:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776544.060099:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776544.060100:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776544.060101:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776544.060102:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72300 x1862022710422400/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776544.060107:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72300 x1862022710422400/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.060110:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776544.060111:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776544.060112:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776544.060112:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776544.060114:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72300 x1862022710422400/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.060117:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776544.060119:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72300 x1862022710422400/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.060123:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776544.060124:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776544.060125:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776544.060126:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776544.060127:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776544.060148:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776544.060150:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776544.060152:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776544.060153:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776544.060155:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72300 x1862022710422400/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776544.060160:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710422400:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776544.060162:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776544.060163:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71180 x1862022710422528/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.060167:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776544.060168:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776544.060169:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776544.060170:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776544.060171:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776544.060173:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710422528:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776544.060175:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776544.060176:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776544.060177:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776544.060178:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776544.060180:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776544.060181:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776544.060183:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776544.060184:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776544.060186:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776544.060187:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710422528, portal 10 00000100:00000001:1.0:1775776544.060188:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776544.060189:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849008512 : -123823860543104 : ffff8f6202c71180) 00000100:00000040:1.0:1775776544.060192:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71180 x1862022710422528/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.060196:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776544.060197:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776544.060199:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776544.060200:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710422528, offset 0 00000400:00000200:1.0:1775776544.060202:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776544.060204:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776544.060206:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776544.060207:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776544.060209:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776544.060211:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776544.060212:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776544.060214:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453853:453853:256:4294967295] 192.168.201.115@tcp LPNI seq info [453778:453778:8:4294967295] 00000400:00000200:1.0:1775776544.060216:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776544.060219:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776544.060222:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776544.060224:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776544.060226:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776544.060227:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775776544.060230:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776544.060231:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1775776544.060233:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776544.060234:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776544.060236:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776544.060238:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776544.060240:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776544.060241:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776544.060243:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776544.060246:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71180 x1862022710422528/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.060258:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776544.060260:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71180 x1862022710422528/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776569 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.060266:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776544.060267:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776544.060270:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776544.060273:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776544.060276:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776544.060278:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308099084 00000100:00000001:1.0:1775776544.060280:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308102918 00000100:00000001:1.0:1775776544.060282:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308099084 00000100:00000001:1.0:1775776544.060284:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308102918 00000100:00000001:1.0:1775776544.060286:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776544.060288:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776544.060290:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776544.060293:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776544/real 1775776544] req@ffff8f6202c71180 x1862022710422528/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776544.060301:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776544.060302:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776544.060304:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776544.060306:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776544.060307:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776544.060310:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776544.060312:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71180 x1862022710422528/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776544.060320:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71180 x1862022710422528/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.060326:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776544.060327:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776544.060329:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776544.060330:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776544.060333:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71180 x1862022710422528/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.060348:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776544.060351:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71180 x1862022710422528/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.060358:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776544.060360:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776544.060361:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776544.060363:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776544.060364:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776544.060366:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776544.060368:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776544.060371:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776544.060373:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776544.060377:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71180 x1862022710422528/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776544.060384:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710422528:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776544.060386:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776544.060387:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776544.060389:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72300 x1862022710422400/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.060393:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776544.060394:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776544.060396:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776544.060397:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776544.060398:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776544.060400:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776544.060401:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776544.060402:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34400. 02000000:00000001:1.0:1775776544.060404:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776544.060405:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776544.060406:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72300. 00000100:00000001:1.0:1775776544.060407:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776544.060408:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776544.060409:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776544.060411:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71180 x1862022710422528/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776569 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776544.060415:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776544.060416:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776544.060417:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776544.060418:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776544.060418:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776544.060419:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776544.060420:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776544.060421:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34800. 02000000:00000001:1.0:1775776544.060422:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776544.060422:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776544.060423:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71180. 00000100:00000001:1.0:1775776544.060424:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776544.060425:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776544.060425:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776544.060427:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776544.060428:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776544.060429:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776544.060430:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776544.187251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776544.187265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776544.187274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776544.187276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776544.187279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776544.187281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776544.187334:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776544.187339:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776544.187342:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776545.083239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776545.083245:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776545.083247:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776545.083252:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776545.083256:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776545.083260:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776545.083263:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776545.083265:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776545.083272:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776545.211251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776545.211257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776545.211262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776545.211263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776545.211267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776545.211268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776545.211333:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776545.211337:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776545.211340:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776546.107246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776546.107254:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776546.107256:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776546.107262:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776546.107265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776546.107269:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776546.107271:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776546.107273:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776546.107276:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776546.235252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776546.235259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776546.235265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776546.235266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776546.235269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776546.235270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776546.235340:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776546.235346:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776546.235351:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776547.131266:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776547.131273:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776547.131275:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776547.131279:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776547.131283:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776547.131286:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776547.131289:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776547.131291:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776547.131293:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776547.259278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776547.259285:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776547.259288:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776547.259292:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776547.259297:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776547.259300:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776547.259302:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776547.259304:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776547.259309:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776547.259311:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776547.259314:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776547.259316:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776547.259387:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776547.259390:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776547.259393:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776547.963250:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308102918, at 4308102976 00000100:00000040:1.0:1775776547.963250:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308102918, at 4308102976 00000100:00000010:1.0:1775776547.963273:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2f60. 00000100:00000010:0.0:1775776547.963282:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2120. 00000100:00000001:1.0:1775776548.155270:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776548.155277:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776548.155279:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776548.155284:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776548.155287:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776548.155291:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776548.155293:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776548.155295:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776548.155297:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776548.283245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776548.283252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776548.283259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776548.283261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776548.283273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776548.283275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776548.283331:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776548.283335:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776548.283338:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776549.179265:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776549.179272:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776549.179274:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776549.179280:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776549.179285:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776549.179289:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776549.179291:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776549.179294:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776549.179300:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776549.179301:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776549.179310:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776549.179314:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776549.179318:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776549.179323:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776549.179331:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13431 00000400:00000200:3.0:1775776549.179340:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776549.179346:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776549.179349:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776549.179352:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776549.179355:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776549.179357:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776549.179372:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776549.179376:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776549.179378:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776549.179380:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776549.179383:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776549.179386:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776549.179388:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776549.179391:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776549.179393:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776549.179395:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776549.179397:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776549.179399:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776549.179402:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776549.179405:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776549.179408:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776549.179411:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776549.179414:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776549.179416:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776549.179419:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776549.179433:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73b80. 00000020:00000040:3.0:1775776549.179437:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776549.179441:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776549.179442:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776549.179445:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776549.179448:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776549.179452:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776549.179459:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000100:00000001:3.0:1775776549.179465:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776549.179473:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73b80 x1862022710422656/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776549.179483:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776549.179488:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776549.179492:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c52008f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73b80 x1862022710422656/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776549.179514:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776549.179518:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776549.179521:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776549.179522:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776549.179523:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776549.179526:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776549.179529:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13431 00000400:00000200:3.0:1775776549.179534:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776549.179539:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:1.0:1775776549.179541:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1775776549.179542:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776549.179545:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776549.179549:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776549.179549:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776549.179554:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776549.179557:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776549.179563:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:1.0:1775776549.179564:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73b80 x1862022710422656/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775776549.179566:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776549.179569:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776549.179587:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776549.179589:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776549.179592:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776549.179604:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776549.179607:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776549.179610:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776549.179614:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776549.179617:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776549.179619:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776549.179620:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1775776549.179622:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776549.179623:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1775776549.179631:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:1.0:1775776549.179632:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776549.179642:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710422656:192.168.201.115@tcp:250:kworker.0 00000100:00080000:3.0:1775776549.179644:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776549.179650:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1775776549.179651:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776549.179655:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776549.179655:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776549.179660:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776549.179663:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1775776549.179665:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776549.179669:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776549.179671:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000010:1.0:1775776549.179672:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776549.179675:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775776549.179678:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1775776549.179682:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776549.179686:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000008:00000001:3.0:1775776549.179692:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776549.179693:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000001:3.0:1775776549.179697:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776549.179702:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710422656, portal 25 00000100:00000001:1.0:1775776549.179705:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000010:3.0:1775776549.179705:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000100:00000001:1.0:1775776549.179709:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849019264 : -123823860532352 : ffff8f6202c73b80) 00000020:00000040:3.0:1775776549.179714:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000040:1.0:1775776549.179718:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73b80 x1862022710422656/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775776549.179720:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776549.179724:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776549.179732:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775776549.179737:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1775776549.179737:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1775776549.179743:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 02000000:00000001:3.0:1775776549.179743:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776549.179748:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776549.179751:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710422656, offset 0 02000000:00000010:3.0:1775776549.179751:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000400:00000200:1.0:1775776549.179757:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776549.179757:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776549.179766:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00100000:3.0:1775776549.179767:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710422784/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776549.179776:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776549.179784:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1775776549.179786:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775776549.179789:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:3.0:1775776549.179791:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776549.179795:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776549.179801:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000040:3.0:1775776549.179805:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710422784/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776549.179807:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453854:453854:256:4294967295] 192.168.201.115@tcp LPNI seq info [453779:453779:8:4294967295] 00000400:00000200:1.0:1775776549.179819:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776549.179838:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:3.0:1775776549.179840:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776549.179847:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00080000:3.0:1775776549.179848:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:1.0:1775776549.179855:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000100:00080000:3.0:1775776549.179856:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776549.179861:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776549.179863:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.0:1775776549.179870:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775776549.179874:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000800:00000200:1.0:1775776549.179875:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000100:00080000:3.0:1775776549.179882:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13431 00000800:00000200:1.1:1775776549.179888:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776549.179892:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000400:00000200:3.0:1775776549.179896:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000100:1.0:1775776549.179903:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776549.179905:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:3.0:1775776549.179906:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776549.179911:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1775776549.179913:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:3.0:1775776549.179917:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776549.179922:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1775776549.179923:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776549.179928:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776549.179931:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776549.179933:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776549.179934:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000400:00000200:1.0:1775776549.179936:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000020:00000001:3.0:1775776549.179936:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776549.179940:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000400:00000010:1.0:1775776549.179942:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000020:00000001:3.0:1775776549.179945:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776549.179950:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000001:1.0:1775776549.179951:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1775776549.179955:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000200:1.0:1775776549.179956:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73b80 x1862022710422656/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776549.179962:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776549.179974:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776549.179977:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1775776549.179978:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:1.0:1775776549.179983:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73b80 x1862022710422656/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776574 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775776549.179985:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00000001:1.0:1775776549.179996:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776549.179997:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776549.179998:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1775776549.180003:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776549.180005:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776549.180008:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776549.180012:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776549.180014:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776549.180015:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308108038 02000000:00000001:3.0:1775776549.180022:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776549.180023:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2f60. 00000100:00000001:3.0:1775776549.180025:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776549.180029:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775776549.180031:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000008:00000001:3.0:1775776549.180033:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776549.180037:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776549.180038:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308108038 00000100:00000010:3.0:1775776549.180041:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000100:00000010:1.0:1775776549.180042:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2a80. 00000100:00000001:1.0:1775776549.180051:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000020:00000040:3.0:1775776549.180051:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776549.180056:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776549.180057:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776549.180062:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776549.180063:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775776549.180069:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776549.180070:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775776549.180076:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1775776549.180077:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776549.180082:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776549/real 1775776549] req@ffff8f6202c73b80 x1862022710422656/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1775776549.180085:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775776549.180094:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1775776549.180104:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:3.0:1775776549.180106:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710422912/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1775776549.180107:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776549.180111:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776549.180117:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776549.180124:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776549.180129:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1775776549.180131:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:1.0:1775776549.180152:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73b80 x1862022710422656/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776549.180159:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73b80 x1862022710422656/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776549.180166:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776549.180167:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776549.180170:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776549.180172:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776549.180175:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73b80 x1862022710422656/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776549.180182:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776549.180187:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73b80 x1862022710422656/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776549.180195:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776549.180198:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776549.180200:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776549.180201:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776549.180204:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776549.180206:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776549.180210:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776549.180213:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776549.180215:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776549.180218:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73b80 x1862022710422656/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776549.180223:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710422656:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776549.180227:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776549.180229:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776549.180231:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73b80 x1862022710422656/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776549.180236:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776549.180239:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776549.180241:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776549.180244:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776549.180245:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776549.180247:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776549.180248:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776549.180251:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36000. 00000100:00000001:3.0:1775776549.180255:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776549.180257:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776549.180261:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776549.180262:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73b80. 00000100:00000001:1.0:1775776549.180265:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776549.180267:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776549.180273:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776549.180273:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710422912/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776549.180281:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776549.180285:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776549.180290:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776549.180293:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776549.180295:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776549.180299:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776549.180303:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71180 x1862022710422784/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776549.180305:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776549.180321:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776549.180328:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776549.180329:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776549.180333:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776549.180339:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1775776549.180341:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 02000000:00000001:1.0:1775776549.180343:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776549.180346:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776549.180350:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710422784:192.168.201.115@tcp:38:kworker.0 00000100:00000040:3.0:1775776549.180350:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13441) 00000100:00000001:1.0:1775776549.180358:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776549.180360:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776549.180362:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776549.180364:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776549.180367:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776549.180368:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776549.180372:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776549.180374:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776549.180377:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776549.180379:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710422784, portal 10 00000100:00000001:1.0:1775776549.180380:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776549.180381:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849008512 : -123823860543104 : ffff8f6202c71180) 00000100:00000040:1.0:1775776549.180386:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71180 x1862022710422784/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776549.180391:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776549.180392:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776549.180393:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776549.180395:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710422784, offset 0 00000400:00000200:1.0:1775776549.180397:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776549.180399:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776549.180402:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776549.180404:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776549.180406:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776549.180408:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776549.180409:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776549.180412:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453855:453855:256:4294967295] 192.168.201.115@tcp LPNI seq info [453780:453780:8:4294967295] 00000400:00000200:1.0:1775776549.180414:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776549.180418:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776549.180421:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776549.180423:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776549.180425:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776549.180427:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775776549.180430:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776549.180432:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000100:1.0:1775776549.180435:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776549.180437:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776549.180438:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776549.180441:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776549.180443:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776549.180444:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776549.180446:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776549.180448:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71180 x1862022710422784/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776549.180452:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776549.180454:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71180 x1862022710422784/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776574 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776549.180458:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776549.180459:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776549.180461:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776549.180464:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776549.180466:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776549.180467:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308104204 00000100:00000001:1.0:1775776549.180470:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308108038 00000100:00000001:1.0:1775776549.180471:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308104204 00000100:00000001:1.0:1775776549.180473:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308108038 00000100:00000001:1.0:1775776549.180474:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776549.180475:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776549.180476:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776549.180478:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776549/real 1775776549] req@ffff8f6202c71180 x1862022710422784/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776549.180484:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776549.180485:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776549.180487:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776549.180488:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776549.180489:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776549.180491:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776549.180494:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71180 x1862022710422784/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776549.180501:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71180 x1862022710422784/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776549.180508:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776549.180509:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776549.180511:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776549.180513:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776549.180516:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71180 x1862022710422784/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776549.180523:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776549.180527:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71180 x1862022710422784/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776549.180533:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776549.180535:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776549.180536:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776549.180537:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776549.180538:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776549.180544:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776549.180546:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776549.180549:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776549.180551:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776549.180555:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71180 x1862022710422784/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776549.180563:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710422784:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776549.180567:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776549.180568:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776549.180571:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71180 x1862022710422784/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776549.180591:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776549.180592:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776549.180594:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776549.180597:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776549.180598:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776549.180599:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776549.180602:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776549.180603:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36400. 02000000:00000001:1.0:1775776549.180604:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776549.180605:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776549.180606:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71180. 00000100:00000001:1.0:1775776549.180608:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776549.180609:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776549.180610:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776549.180612:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776549.180613:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776549.180615:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776549.180616:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776549.180617:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776549.180620:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72300 x1862022710422912/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776549.180624:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776549.180625:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776549.180627:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776549.180628:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776549.180629:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776549.180631:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710422912:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776549.180632:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776549.180634:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776549.180635:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776549.180636:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776549.180638:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776549.180639:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776549.180641:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776549.180643:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776549.180645:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776549.180647:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710422912, portal 10 00000100:00000001:1.0:1775776549.180648:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776549.180649:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849012992 : -123823860538624 : ffff8f6202c72300) 00000100:00000040:1.0:1775776549.180652:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72300 x1862022710422912/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776549.180656:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776549.180657:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776549.180658:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776549.180659:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710422912, offset 0 00000400:00000200:1.0:1775776549.180662:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776549.180664:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776549.180666:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776549.180667:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776549.180669:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776549.180670:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776549.180672:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776549.180674:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453856:453856:256:4294967295] 192.168.201.115@tcp LPNI seq info [453781:453781:8:4294967295] 00000400:00000200:1.0:1775776549.180676:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776549.180680:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776549.180683:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776549.180684:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776549.180687:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776549.180688:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1775776549.180691:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776549.180692:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 28 secs later) 00000800:00000100:1.0:1775776549.180694:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776549.180696:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776549.180697:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776549.180699:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776549.180701:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776549.180702:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776549.180704:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776549.180705:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72300 x1862022710422912/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776549.180710:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776549.180711:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72300 x1862022710422912/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776574 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776549.180715:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776549.180716:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776549.180717:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776549.180719:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776549.180721:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776549.180722:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308104204 00000100:00000001:1.0:1775776549.180723:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308108038 00000100:00000001:1.0:1775776549.180724:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308104204 00000100:00000001:1.0:1775776549.180726:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308108038 00000100:00000001:1.0:1775776549.180727:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776549.180728:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776549.180729:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776549.180731:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776549/real 1775776549] req@ffff8f6202c72300 x1862022710422912/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776549.180735:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776549.180736:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776549.180737:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776549.180738:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776549.180739:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776549.180740:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776549.180741:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72300 x1862022710422912/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776549.180746:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72300 x1862022710422912/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776549.180749:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776549.180750:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776549.180751:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776549.180752:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776549.180753:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72300 x1862022710422912/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776549.180757:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776549.180758:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72300 x1862022710422912/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776549.180762:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776549.180763:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776549.180764:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776549.180765:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776549.180765:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776549.180766:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776549.180768:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776549.180771:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776549.180772:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776549.180774:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72300 x1862022710422912/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776549.180778:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710422912:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776549.180780:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776549.180781:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776549.180783:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72300 x1862022710422912/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776574 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776549.180786:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776549.180786:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776549.180787:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776549.180789:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776549.180789:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776549.180790:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776549.180791:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776549.180792:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34c00. 02000000:00000001:1.0:1775776549.180793:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776549.180794:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776549.180795:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72300. 00000100:00000001:1.0:1775776549.180797:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776549.180798:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776549.180798:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776549.180800:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776549.180801:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776549.180802:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776549.180802:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776549.307277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776549.307282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776549.307288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776549.307293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776549.307297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776549.307298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776549.307356:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776549.307361:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776549.307364:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776550.203270:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776550.203277:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776550.203279:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776550.203285:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776550.203288:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776550.203292:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776550.203294:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776550.203296:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776550.203299:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776550.331301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776550.331308:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776550.331315:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776550.331316:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776550.331320:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776550.331322:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776550.331391:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776550.331395:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776550.331398:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776551.227217:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776551.227225:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776551.227227:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776551.227233:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776551.227238:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776551.227243:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776551.227246:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776551.227248:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776551.227251:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776551.355275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776551.355283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776551.355290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776551.355295:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776551.355298:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776551.355303:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776551.355306:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776551.355309:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776551.355311:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776551.355313:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776551.355316:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776551.355317:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776551.355318:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776551.355319:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776551.355320:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776551.355321:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776551.355323:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776551.355324:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776551.355353:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776551.355357:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776551.355359:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776552.251239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776552.251248:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776552.251250:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776552.251256:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776552.251260:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776552.251263:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776552.251266:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776552.251268:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776552.251270:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776552.379270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776552.379276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776552.379280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776552.379281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776552.379284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776552.379284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776552.379339:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776552.379342:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776552.379344:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776553.083256:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308108038, at 4308108096 00000100:00000040:1.0:1775776553.083256:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308108038, at 4308108096 00000100:00000010:1.0:1775776553.083273:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2a80. 00000100:00000010:0.0:1775776553.083277:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2f60. 00000100:00000001:1.0:1775776553.275248:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776553.275255:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776553.275257:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776553.275262:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776553.275265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776553.275269:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776553.275271:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776553.275273:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776553.275282:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776553.403227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776553.403234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776553.403239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776553.403240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776553.403243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776553.403244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776553.403296:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776553.403299:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776553.403302:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776554.299244:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775776554.299244:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776554.299250:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776554.299253:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776554.299254:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776554.299258:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1775776554.299259:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:3.0:1775776554.299261:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776554.299263:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776554.299265:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776554.299267:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776554.299270:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775776554.299273:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13436 00000100:00000001:1.0:1775776554.299274:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776554.299278:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776554.299284:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776554.299290:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776554.299294:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776554.299297:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776554.299300:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776554.299302:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776554.299304:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776554.299307:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776554.299308:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776554.299311:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776554.299313:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776554.299314:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776554.299316:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776554.299317:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776554.299318:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776554.299319:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776554.299320:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776554.299322:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776554.299325:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776554.299326:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776554.299329:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776554.299331:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776554.299333:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776554.299335:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776554.299337:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776554.299347:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775776554.299349:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776554.299353:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776554.299354:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776554.299364:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776554.299367:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776554.299371:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776554.299380:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000100:00000001:3.0:1775776554.299385:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776554.299394:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710423040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776554.299402:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776554.299406:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776554.299409:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710423040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776554.299426:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776554.299429:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776554.299433:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776554.299434:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776554.299436:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776554.299439:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776554.299441:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13436 00000400:00000200:3.0:1775776554.299445:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776554.299448:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776554.299450:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776554.299453:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776554.299455:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776554.299457:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776554.299459:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776554.299461:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775776554.299462:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1775776554.299462:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776554.299464:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775776554.299465:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1775776554.299466:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:1.0:1775776554.299468:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1775776554.299468:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776554.299470:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776554.299473:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000040:1.0:1775776554.299474:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72a00 x1862022710423040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776554.299475:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776554.299477:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776554.299479:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776554.299481:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000001:1.0:1775776554.299482:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775776554.299497:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776554.299499:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776554.299500:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776554.299506:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776554.299506:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776554.299508:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776554.299510:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776554.299511:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776554.299513:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1775776554.299514:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775776554.299516:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775776554.299517:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776554.299519:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710423040:192.168.201.115@tcp:250:kworker.0 00000100:00000001:3.0:1775776554.299519:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776554.299522:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000100:00000001:1.0:1775776554.299524:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1775776554.299524:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776554.299527:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776554.299528:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776554.299529:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1775776554.299530:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776554.299531:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776554.299532:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776554.299533:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776554.299535:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1775776554.299536:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775776554.299539:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776554.299541:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776554.299542:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776554.299544:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710423168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776554.299548:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776554.299551:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1775776554.299552:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000001:3.0:1775776554.299554:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776554.299557:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710423168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1775776554.299559:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000001:3.0:1775776554.299565:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776554.299567:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1775776554.299568:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710423040, portal 25 00000100:00000001:1.0:1775776554.299570:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1775776554.299570:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1775776554.299572:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849014784 : -123823860536832 : ffff8f6202c72a00) 00000100:00000001:3.0:1775776554.299572:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776554.299573:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776554.299575:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000040:1.0:1775776554.299578:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72a00 x1862022710423040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775776554.299578:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13436 00000400:00000200:3.0:1775776554.299582:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776554.299584:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:1.0:1775776554.299585:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1775776554.299586:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776554.299588:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776554.299589:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000010:1.0:1775776554.299591:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000001:3.0:1775776554.299591:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776554.299593:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000200:1.0:1775776554.299594:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710423040, offset 0 00000100:00000001:3.0:1775776554.299595:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776554.299597:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776554.299599:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776554.299600:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:1.0:1775776554.299601:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000020:00000040:3.0:1775776554.299602:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776554.299603:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776554.299605:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776554.299607:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776554.299609:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776554.299609:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776554.299611:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776554.299613:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776554.299615:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000020:00000040:3.0:1775776554.299615:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776554.299618:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776554.299620:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776554.299621:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1775776554.299622:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776554.299624:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775776554.299625:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 02000000:00000001:3.0:1775776554.299626:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776554.299628:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775776554.299629:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000008:00100000:3.0:1775776554.299630:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1775776554.299632:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000008:00000001:3.0:1775776554.299633:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776554.299634:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776554.299637:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453857:453857:256:4294967295] 192.168.201.115@tcp LPNI seq info [453782:453782:8:4294967295] 00000100:00000010:3.0:1775776554.299637:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000020:00000040:3.0:1775776554.299639:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:1.0:1775776554.299641:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776554.299642:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776554.299643:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776554.299644:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776554.299645:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776554.299647:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776554.299650:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000400:00000200:1.0:1775776554.299652:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:3.0:1775776554.299652:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776554.299656:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710423296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1775776554.299659:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776554.299663:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000010:1.0:1775776554.299665:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000100:00000001:3.0:1775776554.299665:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776554.299668:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710423296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1775776554.299672:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775776554.299675:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776554.299676:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000100:00000040:3.0:1775776554.299678:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776554.299681:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1775776554.299682:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776554.299684:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775776554.299685:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000100:00000040:3.0:1775776554.299687:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000100:1.0:1775776554.299689:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776554.299692:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000100:00000040:3.0:1775776554.299692:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13447) 00000400:00000200:1.0:1775776554.299698:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776554.299703:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776554.299709:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776554.299712:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776554.299719:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776554.299723:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72a00 x1862022710423040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.299731:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776554.299733:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72a00 x1862022710423040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776579 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.299738:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776554.299739:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776554.299744:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776554.299747:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776554.299749:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308113158 00000100:00000010:1.0:1775776554.299753:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2a80. 00000100:00000001:1.0:1775776554.299759:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776554.299763:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308113158 00000100:00000010:1.0:1775776554.299765:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2720. 00000100:00000001:1.0:1775776554.299767:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776554.299769:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776554.299771:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776554.299773:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776554.299777:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776554/real 1775776554] req@ffff8f6202c72a00 x1862022710423040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776554.299786:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776554.299788:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776554.299789:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776554.299792:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776554.299793:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776554.299796:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776554.299799:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72a00 x1862022710423040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776554.299808:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72a00 x1862022710423040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.299815:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776554.299816:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776554.299819:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776554.299821:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776554.299825:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72a00 x1862022710423040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.299832:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776554.299836:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72a00 x1862022710423040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.299840:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776554.299842:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776554.299843:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776554.299844:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776554.299846:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776554.299848:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776554.299849:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776554.299852:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776554.299853:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776554.299856:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72a00 x1862022710423040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776554.299868:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710423040:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776554.299872:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776554.299873:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776554.299875:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72a00 x1862022710423040/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.299879:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776554.299881:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776554.299883:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776554.299885:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776554.299887:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776554.299888:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776554.299889:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776554.299891:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36400. 02000000:00000001:1.0:1775776554.299895:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776554.299897:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776554.299898:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72a00. 00000100:00000001:1.0:1775776554.299901:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776554.299902:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776554.299903:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776554.299909:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776554.299910:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776554.299911:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776554.299912:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776554.299913:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776554.299915:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70700 x1862022710423168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.299920:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776554.299921:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776554.299922:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776554.299924:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776554.299925:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776554.299926:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710423168:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776554.299928:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776554.299929:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776554.299930:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776554.299932:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776554.299934:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776554.299935:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776554.299938:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776554.299939:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776554.299942:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776554.299944:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710423168, portal 10 00000100:00000001:1.0:1775776554.299945:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776554.299946:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849005824 : -123823860545792 : ffff8f6202c70700) 00000100:00000040:1.0:1775776554.299950:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70700 x1862022710423168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.299954:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776554.299956:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776554.299958:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776554.299959:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710423168, offset 0 00000400:00000200:1.0:1775776554.299962:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776554.299964:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776554.299966:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776554.299968:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776554.299970:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776554.299972:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776554.299973:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776554.299975:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453858:453858:256:4294967295] 192.168.201.115@tcp LPNI seq info [453783:453783:8:4294967295] 00000400:00000200:1.0:1775776554.299978:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776554.299981:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776554.299984:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776554.299991:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776554.299994:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776554.299996:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775776554.299999:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776554.300000:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775776554.300003:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776554.300004:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776554.300006:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776554.300008:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776554.300010:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776554.300011:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776554.300013:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776554.300015:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70700 x1862022710423168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.300020:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776554.300021:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70700 x1862022710423168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776579 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.300025:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776554.300026:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776554.300029:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776554.300032:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776554.300034:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776554.300035:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308109323 00000100:00000001:1.0:1775776554.300037:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308113158 00000100:00000001:1.0:1775776554.300038:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308109323 00000100:00000001:1.0:1775776554.300040:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308113158 00000100:00000001:1.0:1775776554.300041:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776554.300042:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776554.300043:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776554.300045:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776554/real 1775776554] req@ffff8f6202c70700 x1862022710423168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776554.300049:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776554.300050:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776554.300051:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776554.300052:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776554.300053:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776554.300054:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776554.300055:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70700 x1862022710423168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776554.300060:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70700 x1862022710423168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.300063:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776554.300064:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776554.300065:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776554.300066:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776554.300068:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70700 x1862022710423168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.300074:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776554.300078:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70700 x1862022710423168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.300085:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776554.300087:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776554.300088:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776554.300089:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776554.300091:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776554.300097:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776554.300099:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776554.300103:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776554.300105:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776554.300109:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70700 x1862022710423168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776554.300117:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710423168:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776554.300121:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776554.300124:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70a80 x1862022710423296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.300129:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776554.300130:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776554.300131:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776554.300132:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776554.300149:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776554.300151:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710423296:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776554.300153:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776554.300155:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776554.300156:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776554.300157:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776554.300159:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776554.300160:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776554.300163:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776554.300164:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776554.300167:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776554.300169:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710423296, portal 10 00000100:00000001:1.0:1775776554.300170:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776554.300171:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849006720 : -123823860544896 : ffff8f6202c70a80) 00000100:00000040:1.0:1775776554.300174:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70a80 x1862022710423296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.300179:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776554.300180:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776554.300181:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776554.300183:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710423296, offset 0 00000400:00000200:1.0:1775776554.300185:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776554.300187:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776554.300189:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776554.300191:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776554.300192:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776554.300194:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776554.300196:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776554.300198:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453859:453859:256:4294967295] 192.168.201.115@tcp LPNI seq info [453784:453784:8:4294967295] 00000400:00000200:1.0:1775776554.300200:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776554.300204:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776554.300206:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776554.300208:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776554.300210:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776554.300212:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775776554.300215:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776554.300216:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775776554.300219:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776554.300220:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776554.300222:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776554.300224:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776554.300226:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776554.300227:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776554.300229:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776554.300231:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70a80 x1862022710423296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.300235:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776554.300237:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70a80 x1862022710423296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776579 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.300241:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776554.300242:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776554.300243:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776554.300245:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776554.300246:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776554.300247:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308109324 00000100:00000001:1.0:1775776554.300249:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308113158 00000100:00000001:1.0:1775776554.300250:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308109324 00000100:00000001:1.0:1775776554.300252:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308113158 00000100:00000001:1.0:1775776554.300253:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776554.300254:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776554.300255:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776554.300257:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776554/real 1775776554] req@ffff8f6202c70a80 x1862022710423296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776554.300261:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776554.300262:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776554.300263:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776554.300264:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776554.300265:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776554.300266:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776554.300268:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70a80 x1862022710423296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776554.300272:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70a80 x1862022710423296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.300277:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776554.300278:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776554.300279:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776554.300280:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776554.300281:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70a80 x1862022710423296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.300285:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776554.300287:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70a80 x1862022710423296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.300290:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776554.300291:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776554.300292:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776554.300293:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776554.300294:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776554.300295:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776554.300296:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776554.300297:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776554.300298:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776554.300300:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70a80 x1862022710423296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776554.300304:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710423296:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776554.300306:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776554.300307:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776554.300309:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70700 x1862022710423168/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.300313:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776554.300315:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776554.300316:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776554.300317:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776554.300318:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776554.300319:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776554.300320:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776554.300321:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34c00. 02000000:00000001:1.0:1775776554.300323:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776554.300324:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776554.300325:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70700. 00000100:00000001:1.0:1775776554.300326:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776554.300327:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776554.300328:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776554.300329:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70a80 x1862022710423296/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776579 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776554.300333:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776554.300333:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776554.300334:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776554.300336:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776554.300337:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776554.300337:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776554.300338:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776554.300339:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36000. 02000000:00000001:1.0:1775776554.300340:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776554.300340:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776554.300341:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70a80. 00000100:00000001:1.0:1775776554.300342:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776554.300343:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776554.300344:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776554.300345:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776554.300346:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776554.300347:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776554.300348:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776554.427269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776554.427275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776554.427279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776554.427281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776554.427283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776554.427284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776554.427348:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776554.427352:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776554.427356:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776555.323243:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776555.323251:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776555.323253:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776555.323260:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776555.323264:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776555.323269:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776555.323272:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776555.323274:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776555.323277:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776555.451304:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776555.451311:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776555.451317:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776555.451318:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776555.451322:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776555.451323:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776555.451392:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776555.451397:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776555.451400:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776556.347231:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776556.347238:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776556.347239:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776556.347244:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776556.347248:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776556.347251:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776556.347253:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776556.347255:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776556.347258:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776556.475277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776556.475285:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776556.475288:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776556.475291:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776556.475297:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776556.475299:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776556.475301:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776556.475304:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776556.475307:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776556.475309:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776556.475313:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776556.475314:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776556.475383:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776556.475387:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776556.475391:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776557.371272:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776557.371281:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776557.371290:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776557.371298:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776557.371302:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776557.371307:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776557.371309:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776557.371313:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776557.371315:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776557.499276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776557.499281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776557.499287:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776557.499288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776557.499292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776557.499293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776557.499354:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776557.499358:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776557.499362:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776558.203264:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308113158, at 4308113216 00000100:00000040:1.0:1775776558.203264:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308113158, at 4308113216 00000100:00000010:0.0:1775776558.203280:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2a80. 00000100:00000010:1.0:1775776558.203282:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2720. 00000100:00000001:1.0:1775776558.395285:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776558.395292:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776558.395293:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776558.395298:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776558.395302:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776558.395305:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776558.395308:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776558.395309:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776558.395312:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776558.523260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776558.523266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776558.523273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776558.523274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776558.523278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776558.523279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776558.523351:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776558.523355:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776558.523358:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776559.419257:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776559.419262:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775776559.419268:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776559.419269:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776559.419272:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776559.419273:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776559.419277:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775776559.419278:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776559.419283:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776559.419284:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776559.419288:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776559.419292:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775776559.419293:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13442 00000100:00000001:1.0:1775776559.419296:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776559.419299:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776559.419303:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776559.419310:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776559.419313:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776559.419316:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776559.419320:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776559.419322:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776559.419325:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776559.419328:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776559.419331:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776559.419334:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776559.419336:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776559.419339:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776559.419341:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776559.419344:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776559.419346:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776559.419348:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776559.419349:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776559.419352:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776559.419354:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776559.419357:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776559.419359:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776559.419362:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776559.419364:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776559.419366:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776559.419368:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776559.419377:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775776559.419380:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776559.419384:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776559.419385:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776559.419388:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776559.419390:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776559.419393:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776559.419413:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000100:00000001:3.0:1775776559.419420:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776559.419429:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710423424/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776559.419438:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776559.419443:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776559.419448:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710423424/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776559.419468:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776559.419471:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776559.419474:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776559.419475:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776559.419477:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776559.419479:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776559.419481:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13442 00000400:00000200:3.0:1775776559.419485:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776559.419488:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776559.419491:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776559.419494:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776559.419496:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776559.419498:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1775776559.419499:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776559.419502:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:1.0:1775776559.419503:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1775776559.419505:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775776559.419506:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1775776559.419507:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776559.419510:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776559.419512:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000040:1.0:1775776559.419515:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71500 x1862022710423424/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775776559.419515:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776559.419520:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776559.419527:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776559.419531:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776559.419532:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1775776559.419534:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776559.419536:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776559.419538:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000001:1.0:1775776559.419542:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776559.419543:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776559.419547:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776559.419548:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776559.419550:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776559.419552:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776559.419555:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776559.419556:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1775776559.419561:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710423424:192.168.201.115@tcp:250:kworker.0 02000000:00000001:3.0:1775776559.419561:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776559.419565:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776559.419569:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00100000:3.0:1775776559.419570:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775776559.419573:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:3.0:1775776559.419575:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776559.419576:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776559.419578:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776559.419581:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:3.0:1775776559.419583:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775776559.419586:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000010:1.0:1775776559.419587:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:3.0:1775776559.419588:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776559.419590:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776559.419592:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776559.419594:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776559.419596:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776559.419598:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776559.419600:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000010:3.0:1775776559.419603:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000400:00000010:1.0:1775776559.419605:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000001:3.0:1775776559.419607:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776559.419613:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710423552/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1775776559.419614:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776559.419622:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710423424, portal 25 00000100:00000001:3.0:1775776559.419623:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776559.419627:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776559.419628:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776559.419630:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849009408 : -123823860542208 : ffff8f6202c71500) 00000100:00000040:3.0:1775776559.419634:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710423552/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776559.419640:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71500 x1862022710423424/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.419664:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776559.419669:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776559.419671:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776559.419672:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00080000:3.0:1775776559.419674:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1775776559.419676:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710423424, offset 0 00000100:00080000:3.0:1775776559.419678:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776559.419681:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776559.419683:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775776559.419688:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776559.419688:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776559.419694:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13442 00000400:00000200:1.0:1775776559.419700:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776559.419701:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776559.419708:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000400:00000200:1.0:1775776559.419710:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00080000:3.0:1775776559.419712:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1775776559.419717:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1775776559.419718:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776559.419721:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776559.419724:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000040:3.0:1775776559.419725:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776559.419729:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776559.419731:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000020:00000001:3.0:1775776559.419733:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776559.419735:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000400:00000200:1.0:1775776559.419736:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000020:00000001:3.0:1775776559.419738:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776559.419741:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000400:00000200:1.0:1775776559.419744:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453860:453860:256:4294967295] 192.168.201.115@tcp LPNI seq info [453785:453785:8:4294967295] 00000020:00000001:3.0:1775776559.419745:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776559.419750:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776559.419755:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775776559.419756:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776559.419760:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776559.419761:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776559.419765:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776559.419771:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000020:00000040:3.0:1775776559.419771:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776559.419777:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776559.419780:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776559.419783:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 02000000:00000001:3.0:1775776559.419784:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776559.419788:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:1.0:1775776559.419791:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 02000000:00000001:3.0:1775776559.419791:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776559.419795:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776559.419797:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776559.419801:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776559.419802:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775776559.419803:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776559.419807:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00000010:3.0:1775776559.419809:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775776559.419812:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776559.419817:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000800:00000200:1.1:1775776559.419817:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:3.0:1775776559.419819:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776559.419822:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00000001:3.0:1775776559.419823:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776559.419825:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776559.419829:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1775776559.419833:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 02000000:00000010:3.0:1775776559.419835:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000800:00000010:1.0:1775776559.419836:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000100:00000001:3.0:1775776559.419840:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776559.419845:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00100000:3.0:1775776559.419847:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710423680/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776559.419854:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776559.419863:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000100:00000001:3.0:1775776559.419863:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776559.419866:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776559.419868:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000040:3.0:1775776559.419872:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710423680/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.419875:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1775776559.419883:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776559.419884:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71500 x1862022710423424/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776559.419887:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776559.419897:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776559.419902:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776559.419903:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776559.419906:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:1.0:1775776559.419910:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71500 x1862022710423424/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776584 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776559.419912:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13452) 00000100:00000001:1.0:1775776559.419921:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776559.419923:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776559.419928:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776559.419933:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776559.419935:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308118278 00000100:00000010:1.0:1775776559.419941:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2720. 00000100:00000001:1.0:1775776559.419947:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776559.419952:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308118278 00000100:00000010:1.0:1775776559.419955:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2600. 00000100:00000001:1.0:1775776559.419958:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776559.419961:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776559.419964:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776559.419966:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776559.419970:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776559/real 1775776559] req@ffff8f6202c71500 x1862022710423424/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776559.419979:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776559.419981:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776559.419983:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776559.419985:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776559.419987:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776559.419989:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776559.419993:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71500 x1862022710423424/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776559.420001:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71500 x1862022710423424/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.420009:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776559.420011:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776559.420013:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776559.420015:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776559.420018:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71500 x1862022710423424/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.420025:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776559.420030:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71500 x1862022710423424/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.420037:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776559.420040:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776559.420041:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776559.420043:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776559.420045:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776559.420047:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776559.420050:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776559.420054:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776559.420056:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776559.420061:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71500 x1862022710423424/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776559.420070:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710423424:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776559.420075:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776559.420077:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776559.420080:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71500 x1862022710423424/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.420087:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776559.420090:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776559.420092:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776559.420095:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776559.420096:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776559.420098:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776559.420100:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776559.420102:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34400. 02000000:00000001:1.0:1775776559.420106:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776559.420108:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776559.420110:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71500. 00000100:00000001:1.0:1775776559.420113:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776559.420114:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776559.420115:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776559.420121:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776559.420122:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776559.420125:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776559.420126:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776559.420127:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776559.420131:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70380 x1862022710423552/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.420156:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776559.420158:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776559.420159:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776559.420161:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776559.420163:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776559.420166:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710423552:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776559.420169:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776559.420171:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776559.420172:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776559.420174:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776559.420177:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776559.420179:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776559.420183:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776559.420185:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776559.420188:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776559.420191:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710423552, portal 10 00000100:00000001:1.0:1775776559.420193:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776559.420194:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004928 : -123823860546688 : ffff8f6202c70380) 00000100:00000040:1.0:1775776559.420201:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70380 x1862022710423552/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.420207:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776559.420209:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776559.420212:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776559.420214:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710423552, offset 0 00000400:00000200:1.0:1775776559.420218:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776559.420223:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776559.420226:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776559.420229:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776559.420232:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776559.420234:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776559.420237:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776559.420240:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453861:453861:256:4294967295] 192.168.201.115@tcp LPNI seq info [453786:453786:8:4294967295] 00000400:00000200:1.0:1775776559.420243:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776559.420250:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776559.420254:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776559.420257:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776559.420261:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776559.420263:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775776559.420267:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776559.420270:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775776559.420274:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776559.420275:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776559.420278:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776559.420281:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776559.420284:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776559.420285:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776559.420288:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776559.420291:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70380 x1862022710423552/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.420299:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776559.420301:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70380 x1862022710423552/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776584 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.420307:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776559.420308:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776559.420312:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776559.420316:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776559.420319:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776559.420321:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308114444 00000100:00000001:1.0:1775776559.420323:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308118278 00000100:00000001:1.0:1775776559.420325:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308114444 00000100:00000001:1.0:1775776559.420327:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308118278 00000100:00000001:1.0:1775776559.420329:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776559.420331:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776559.420333:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776559.420337:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776559/real 1775776559] req@ffff8f6202c70380 x1862022710423552/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776559.420344:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776559.420346:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776559.420348:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776559.420350:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776559.420351:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776559.420353:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776559.420355:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70380 x1862022710423552/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776559.420362:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70380 x1862022710423552/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.420367:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776559.420368:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776559.420370:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776559.420371:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776559.420374:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70380 x1862022710423552/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.420380:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776559.420383:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70380 x1862022710423552/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.420390:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776559.420392:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776559.420394:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776559.420409:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776559.420411:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776559.420415:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776559.420418:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776559.420421:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776559.420423:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776559.420427:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70380 x1862022710423552/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776559.420434:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710423552:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776559.420438:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776559.420440:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73480 x1862022710423680/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.420446:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776559.420447:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776559.420449:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776559.420450:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776559.420451:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776559.420453:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710423680:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776559.420455:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776559.420457:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776559.420458:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776559.420458:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776559.420461:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776559.420462:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776559.420465:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776559.420466:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776559.420468:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776559.420470:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710423680, portal 10 00000100:00000001:1.0:1775776559.420471:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776559.420472:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849017472 : -123823860534144 : ffff8f6202c73480) 00000100:00000040:1.0:1775776559.420476:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73480 x1862022710423680/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.420480:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776559.420481:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776559.420483:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776559.420484:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710423680, offset 0 00000400:00000200:1.0:1775776559.420486:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776559.420488:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776559.420491:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776559.420492:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776559.420494:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776559.420495:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776559.420497:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776559.420499:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453862:453862:256:4294967295] 192.168.201.115@tcp LPNI seq info [453787:453787:8:4294967295] 00000400:00000200:1.0:1775776559.420501:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776559.420505:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776559.420507:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776559.420509:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776559.420511:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776559.420513:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775776559.420515:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776559.420517:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775776559.420519:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776559.420520:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776559.420522:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776559.420525:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776559.420526:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776559.420528:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776559.420529:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776559.420531:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73480 x1862022710423680/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.420536:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776559.420537:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73480 x1862022710423680/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776584 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.420541:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776559.420542:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776559.420543:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776559.420545:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776559.420546:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776559.420548:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308114444 00000100:00000001:1.0:1775776559.420549:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308118278 00000100:00000001:1.0:1775776559.420551:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308114444 00000100:00000001:1.0:1775776559.420553:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308118278 00000100:00000001:1.0:1775776559.420555:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776559.420557:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776559.420559:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776559.420562:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776559/real 1775776559] req@ffff8f6202c73480 x1862022710423680/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776559.420569:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776559.420570:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776559.420571:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776559.420573:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776559.420574:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776559.420576:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776559.420578:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73480 x1862022710423680/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776559.420585:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73480 x1862022710423680/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.420590:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776559.420591:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776559.420593:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776559.420594:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776559.420597:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73480 x1862022710423680/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.420602:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776559.420605:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73480 x1862022710423680/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.420611:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776559.420613:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776559.420614:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776559.420615:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776559.420617:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776559.420618:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776559.420620:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776559.420623:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776559.420624:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776559.420628:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73480 x1862022710423680/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776559.420634:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710423680:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776559.420637:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776559.420638:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776559.420640:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70380 x1862022710423552/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.420645:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776559.420646:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776559.420649:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776559.420651:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776559.420652:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776559.420654:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776559.420655:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776559.420656:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34800. 02000000:00000001:1.0:1775776559.420658:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776559.420659:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776559.420660:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70380. 00000100:00000001:1.0:1775776559.420662:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776559.420663:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776559.420664:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776559.420666:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73480 x1862022710423680/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776584 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776559.420672:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776559.420673:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776559.420674:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776559.420676:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776559.420676:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776559.420677:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776559.420679:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776559.420680:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37000. 02000000:00000001:1.0:1775776559.420682:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776559.420683:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776559.420684:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73480. 00000100:00000001:1.0:1775776559.420686:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776559.420687:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776559.420688:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776559.420691:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776559.420692:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776559.420693:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776559.420695:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776559.547254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776559.547259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776559.547266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776559.547267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776559.547271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776559.547272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776559.547331:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776559.547335:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776559.547337:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776560.443260:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776560.443266:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776560.443268:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776560.443273:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776560.443277:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776560.443280:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776560.443283:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776560.443284:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776560.443286:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776560.571258:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776560.571265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776560.571272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776560.571273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776560.571277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776560.571278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776560.571337:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776560.571343:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776560.571347:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776561.467234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776561.467239:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776561.467241:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776561.467245:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776561.467247:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776561.467249:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776561.467251:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776561.467253:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776561.467254:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776561.595281:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776561.595288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776561.595294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776561.595298:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776561.595300:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776561.595305:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776561.595308:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776561.595311:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776561.595313:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776561.595316:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776561.595320:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776561.595321:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776561.595323:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776561.595324:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776561.595326:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776561.595327:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776561.595329:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776561.595330:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776561.595429:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776561.595435:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776561.595439:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776562.491325:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776562.491332:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776562.491334:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776562.491339:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776562.491356:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776562.491359:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776562.491362:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776562.491364:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776562.491366:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776562.619295:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776562.619301:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776562.619307:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776562.619308:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776562.619312:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776562.619314:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776562.619401:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776562.619404:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776562.619407:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776563.323215:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308118278, at 4308118336 00000100:00000010:0.0:1775776563.323227:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2720. 00000100:00000040:1.0:1775776563.323291:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308118278, at 4308118336 00000100:00000010:1.0:1775776563.323296:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2600. 00000100:00000001:1.0:1775776563.515245:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776563.515250:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776563.515251:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776563.515254:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776563.515257:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776563.515260:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776563.515262:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776563.515263:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776563.515265:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776563.643239:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776563.643243:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776563.643247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776563.643248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776563.643251:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776563.643252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776563.643304:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776563.643307:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776563.643309:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776564.539195:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776564.539204:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776564.539206:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776564.539207:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776564.539210:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776564.539212:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776564.539212:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775776564.539216:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776564.539218:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776564.539219:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776564.539222:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776564.539224:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776564.539227:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775776564.539227:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13447 00000100:00000001:1.0:1775776564.539231:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776564.539238:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776564.539245:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776564.539248:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776564.539251:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776564.539254:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776564.539256:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776564.539259:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776564.539262:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776564.539264:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776564.539266:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776564.539269:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776564.539271:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776564.539274:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776564.539276:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776564.539278:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776564.539280:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776564.539282:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776564.539285:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776564.539288:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776564.539290:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776564.539294:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776564.539297:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776564.539299:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776564.539302:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776564.539304:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776564.539326:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775776564.539330:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776564.539333:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776564.539335:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776564.539338:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776564.539341:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776564.539344:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776564.539351:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775776564.539357:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776564.539365:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710423808/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776564.539374:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776564.539378:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776564.539382:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710423808/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776564.539400:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776564.539403:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776564.539407:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776564.539408:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776564.539409:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776564.539412:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776564.539415:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13447 00000400:00000200:3.0:1775776564.539420:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776564.539422:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1775776564.539423:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:1.0:1775776564.539425:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776564.539427:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1775776564.539427:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776564.539431:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000040:1.0:1775776564.539435:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71c00 x1862022710423808/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776564.539435:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776564.539440:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776564.539445:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776564.539450:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775776564.539451:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776564.539453:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776564.539453:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776564.539456:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:1.0:1775776564.539458:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:3.0:1775776564.539459:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 02000000:00000001:1.0:1775776564.539462:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1775776564.539463:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776564.539467:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775776564.539467:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00100000:1.0:1775776564.539474:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710423808:192.168.201.115@tcp:250:kworker.0 00000100:00000040:3.0:1775776564.539475:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776564.539480:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776564.539481:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1775776564.539482:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776564.539485:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776564.539486:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:3.0:1775776564.539487:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:1.0:1775776564.539490:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776564.539492:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000001:1.0:1775776564.539495:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775776564.539496:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776564.539498:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:1.0:1775776564.539502:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4fc00. 02000000:00000001:3.0:1775776564.539502:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776564.539507:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776564.539507:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776564.539512:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775776564.539516:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000008:00100000:3.0:1775776564.539516:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776564.539520:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776564.539521:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000001:3.0:1775776564.539522:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776564.539528:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000400:00000010:1.0:1775776564.539530:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000020:00000040:3.0:1775776564.539530:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776564.539535:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776564.539536:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776564.539539:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710423808, portal 25 00000100:00000001:3.0:1775776564.539539:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776564.539542:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1775776564.539543:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776564.539545:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849011200 : -123823860540416 : ffff8f6202c71c00) 02000000:00000001:3.0:1775776564.539545:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776564.539551:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000040:1.0:1775776564.539554:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71c00 x1862022710423808/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776589 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776564.539556:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776564.539562:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776564.539566:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00100000:3.0:1775776564.539569:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710423936/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1775776564.539571:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776564.539576:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710423808, offset 0 00000100:00000001:3.0:1775776564.539584:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775776564.539588:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776564.539589:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776564.539594:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710423936/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776564.539599:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776564.539606:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776564.539614:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1775776564.539615:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776564.539619:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00080000:3.0:1775776564.539619:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776564.539624:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:1.0:1775776564.539626:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1775776564.539627:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776564.539630:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775776564.539631:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:3.0:1775776564.539636:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775776564.539639:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453863:453863:256:4294967295] 192.168.201.115@tcp LPNI seq info [453788:453788:8:4294967295] 00000100:00080000:3.0:1775776564.539641:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13447 00000400:00000200:1.0:1775776564.539646:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776564.539649:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776564.539653:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776564.539658:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1775776564.539660:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000040:3.0:1775776564.539664:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000800:00000200:1.0:1775776564.539668:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776564.539668:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776564.539673:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000010:1.0:1775776564.539674:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000100:00000001:3.0:1775776564.539676:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776564.539679:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776564.539682:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000800:00000200:1.0:1775776564.539683:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:3.0:1775776564.539685:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:1.0:1775776564.539687:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000020:00000040:3.0:1775776564.539688:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776564.539692:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:1.1:1775776564.539695:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000001:3.0:1775776564.539697:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000200:1.1:1775776564.539698:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000100:00000040:3.0:1775776564.539702:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000800:00000100:1.0:1775776564.539703:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776564.539707:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000100:00000001:3.0:1775776564.539707:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776564.539712:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:3.0:1775776564.539712:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776564.539716:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776564.539720:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000020:00000040:3.0:1775776564.539720:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776564.539723:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000100:00080000:3.0:1775776564.539725:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000400:00000010:1.0:1775776564.539726:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:3.0:1775776564.539731:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776564.539733:0:2411:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1775776564.539733:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:1.0:1775776564.539737:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71c00 x1862022710423808/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776589 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775776564.539737:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776564.539742:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776564.539746:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776564.539750:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000008:00100000:3.0:1775776564.539750:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000040:1.0:1775776564.539755:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71c00 x1862022710423808/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776589 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1775776564.539755:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776564.539759:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776564.539765:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776564.539765:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000100:00000001:1.0:1775776564.539768:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:3.0:1775776564.539770:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000100:00000001:1.0:1775776564.539774:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776564.539774:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776564.539776:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776564.539779:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776564.539780:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775776564.539782:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308123398 02000000:00000001:3.0:1775776564.539783:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776564.539786:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776564.539787:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2600. 02000000:00000010:3.0:1775776564.539790:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:1.0:1775776564.539793:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1775776564.539794:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776564.539799:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308123398 00000100:00100000:3.0:1775776564.539802:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710424064/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:1.0:1775776564.539803:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2420. 00000100:00000001:1.0:1775776564.539808:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776564.539812:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776564.539817:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776564.539817:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776564.539821:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1775776564.539822:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776564.539826:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776564/real 1775776564] req@ffff8f6202c71c00 x1862022710423808/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776564.539829:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710424064/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776564.539837:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776564.539839:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776564.539841:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776564.539845:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000100:00000001:3.0:1775776564.539845:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776564.539846:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000040:3.0:1775776564.539849:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776564.539850:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776564.539854:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71c00 x1862022710423808/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776564.539855:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776564.539860:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1775776564.539864:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71c00 x1862022710423808/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776564.539866:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776564.539873:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776564.539874:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776564.539874:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13457) 00000100:00000001:1.0:1775776564.539877:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776564.539879:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776564.539882:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71c00 x1862022710423808/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776564.539886:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776564.539890:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71c00 x1862022710423808/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776564.539894:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776564.539896:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776564.539897:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776564.539897:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776564.539899:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776564.539900:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776564.539902:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776564.539904:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776564.539905:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776564.539907:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71c00 x1862022710423808/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776564.539912:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710423808:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776564.539914:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776564.539915:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776564.539916:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71c00 x1862022710423808/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776564.539920:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776564.539922:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776564.539924:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776564.539926:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776564.539927:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776564.539928:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776564.539930:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776564.539931:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37400. 02000000:00000001:1.0:1775776564.539933:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776564.539935:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776564.539936:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71c00. 00000100:00000001:1.0:1775776564.539938:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776564.539939:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776564.539940:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776564.539945:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776564.539946:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776564.539947:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776564.539948:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776564.539950:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776564.539953:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70000 x1862022710423936/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776564.539960:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776564.539961:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776564.539963:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776564.539965:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776564.539967:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776564.539970:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710423936:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776564.539973:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776564.539974:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776564.539976:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776564.539977:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776564.539980:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776564.539982:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776564.539986:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776564.539989:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776564.539992:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776564.539995:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710423936, portal 10 00000100:00000001:1.0:1775776564.539997:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776564.539998:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004032 : -123823860547584 : ffff8f6202c70000) 00000100:00000040:1.0:1775776564.540004:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70000 x1862022710423936/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776589 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776564.540008:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776564.540010:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776564.540012:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776564.540014:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710423936, offset 0 00000400:00000200:1.0:1775776564.540016:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776564.540018:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776564.540021:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776564.540023:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776564.540024:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776564.540026:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776564.540027:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776564.540029:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453864:453864:256:4294967295] 192.168.201.115@tcp LPNI seq info [453789:453789:8:4294967295] 00000400:00000200:1.0:1775776564.540031:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776564.540035:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776564.540037:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776564.540039:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776564.540041:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776564.540043:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775776564.540046:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776564.540047:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1775776564.540049:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776564.540050:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776564.540052:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776564.540054:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776564.540056:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776564.540057:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776564.540058:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776564.540060:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70000 x1862022710423936/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776589 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776564.540065:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776564.540066:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70000 x1862022710423936/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776589 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776564.540070:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776564.540071:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776564.540073:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776564.540075:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776564.540078:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776564.540079:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308119563 00000100:00000001:1.0:1775776564.540082:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308123398 00000100:00000001:1.0:1775776564.540084:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308119563 00000100:00000001:1.0:1775776564.540086:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308123398 00000100:00000001:1.0:1775776564.540087:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776564.540089:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776564.540091:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776564.540094:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776564/real 1775776564] req@ffff8f6202c70000 x1862022710423936/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776564.540102:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776564.540103:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776564.540105:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776564.540107:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776564.540108:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776564.540110:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776564.540113:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70000 x1862022710423936/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776564.540122:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70000 x1862022710423936/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776564.540128:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776564.540129:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776564.540130:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776564.540131:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776564.540148:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70000 x1862022710423936/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776564.540152:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776564.540154:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70000 x1862022710423936/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776564.540159:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776564.540161:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776564.540161:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776564.540162:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776564.540163:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776564.540166:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776564.540168:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776564.540169:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776564.540170:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776564.540172:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70000 x1862022710423936/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776564.540177:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710423936:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776564.540179:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776564.540181:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70e00 x1862022710424064/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776564.540184:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776564.540185:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776564.540186:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776564.540187:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776564.540189:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776564.540190:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710424064:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776564.540192:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776564.540193:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776564.540194:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776564.540195:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776564.540197:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776564.540198:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776564.540200:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776564.540202:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776564.540204:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776564.540205:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710424064, portal 10 00000100:00000001:1.0:1775776564.540206:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776564.540207:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849007616 : -123823860544000 : ffff8f6202c70e00) 00000100:00000040:1.0:1775776564.540211:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70e00 x1862022710424064/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776589 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776564.540215:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776564.540216:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776564.540217:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776564.540218:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710424064, offset 0 00000400:00000200:1.0:1775776564.540220:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776564.540223:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776564.540225:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776564.540226:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776564.540228:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776564.540229:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776564.540230:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776564.540232:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453865:453865:256:4294967295] 192.168.201.115@tcp LPNI seq info [453790:453790:8:4294967295] 00000400:00000200:1.0:1775776564.540235:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776564.540238:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776564.540241:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776564.540243:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776564.540245:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776564.540247:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775776564.540249:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776564.540251:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1775776564.540253:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776564.540254:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776564.540256:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776564.540258:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776564.540259:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776564.540260:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776564.540262:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776564.540264:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70e00 x1862022710424064/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776589 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776564.540268:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776564.540270:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70e00 x1862022710424064/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776589 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776564.540274:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776564.540274:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776564.540276:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776564.540277:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776564.540279:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776564.540280:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308119564 00000100:00000001:1.0:1775776564.540281:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308123398 00000100:00000001:1.0:1775776564.540282:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308119564 00000100:00000001:1.0:1775776564.540284:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308123398 00000100:00000001:1.0:1775776564.540285:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776564.540286:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776564.540287:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776564.540289:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776564/real 1775776564] req@ffff8f6202c70e00 x1862022710424064/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776564.540293:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776564.540293:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776564.540294:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776564.540295:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776564.540296:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776564.540297:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776564.540299:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70e00 x1862022710424064/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776564.540303:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70e00 x1862022710424064/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776564.540319:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776564.540320:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776564.540321:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776564.540322:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776564.540324:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70e00 x1862022710424064/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776564.540328:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776564.540330:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70e00 x1862022710424064/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776564.540333:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776564.540335:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776564.540335:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776564.540336:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776564.540337:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776564.540338:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776564.540340:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776564.540341:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776564.540342:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776564.540344:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70e00 x1862022710424064/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776564.540348:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710424064:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776564.540350:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776564.540351:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776564.540353:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70000 x1862022710423936/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776564.540357:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776564.540358:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776564.540359:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776564.540361:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776564.540362:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776564.540363:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776564.540364:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776564.540365:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35800. 02000000:00000001:1.0:1775776564.540366:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776564.540367:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776564.540368:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70000. 00000100:00000001:1.0:1775776564.540370:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776564.540371:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776564.540372:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776564.540373:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70e00 x1862022710424064/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776589 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776564.540377:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776564.540377:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776564.540378:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776564.540379:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776564.540380:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776564.540380:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776564.540381:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776564.540382:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35400. 02000000:00000001:1.0:1775776564.540383:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776564.540383:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776564.540384:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70e00. 00000100:00000001:1.0:1775776564.540385:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776564.540386:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776564.540387:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776564.540389:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776564.540389:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776564.540390:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776564.540391:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776564.667243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776564.667248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776564.667252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776564.667253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776564.667255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776564.667256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776564.667329:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776564.667333:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776564.667336:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.0:1775776565.371281:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:3.0:1775776565.371289:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:3.0:1775776565.371316:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:1.0:1775776565.371462:0:2405:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000001:1.0:1775776565.371468:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1775776565.371478:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1775776565.371481:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1775776565.371482:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776565.563250:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776565.563257:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776565.563259:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776565.563263:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776565.563266:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776565.563271:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776565.563273:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776565.563276:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776565.563278:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776565.691220:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776565.691226:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776565.691230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776565.691231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776565.691235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776565.691236:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776565.691283:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776565.691308:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776565.691310:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776566.587235:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776566.587241:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776566.587243:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776566.587248:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776566.587252:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776566.587255:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776566.587257:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776566.587260:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776566.587262:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776566.715337:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776566.715346:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776566.715348:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776566.715353:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776566.715359:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776566.715368:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776566.715370:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776566.715373:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776566.715377:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776566.715378:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776566.715381:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776566.715382:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776566.715477:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776566.715483:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776566.715487:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776567.611239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776567.611244:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776567.611245:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776567.611250:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776567.611264:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776567.611267:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776567.611269:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776567.611270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776567.611272:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776567.739276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776567.739282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776567.739286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776567.739287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776567.739290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776567.739290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776567.739342:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776567.739345:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776567.739348:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776568.443227:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308123398, at 4308123456 00000100:00000040:0.0:1775776568.443236:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308123398, at 4308123456 00000100:00000010:1.0:1775776568.443251:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2420. 00000100:00000010:0.0:1775776568.443273:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2600. 00000100:00000001:1.0:1775776568.635276:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776568.635281:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776568.635282:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776568.635287:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776568.635289:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776568.635293:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776568.635295:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776568.635297:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776568.635299:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776568.763260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776568.763266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776568.763271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776568.763272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776568.763276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776568.763277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776568.763334:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776568.763337:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776568.763339:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776569.659214:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776569.659228:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776569.659234:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1775776569.659235:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1775776569.659236:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776569.659238:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1775776569.659240:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:3.0:1775776569.659241:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776569.659244:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776569.659245:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776569.659247:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776569.659251:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775776569.659252:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13452 00000100:00000001:1.0:1775776569.659253:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776569.659256:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776569.659260:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776569.659265:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776569.659267:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776569.659270:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776569.659273:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776569.659275:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776569.659276:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776569.659279:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776569.659280:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776569.659282:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776569.659283:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776569.659285:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776569.659286:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776569.659288:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776569.659289:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776569.659290:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776569.659291:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776569.659293:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776569.659295:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776569.659296:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776569.659299:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776569.659301:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776569.659302:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776569.659304:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776569.659306:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776569.659312:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73100. 00000020:00000040:3.0:1775776569.659315:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776569.659318:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776569.659319:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776569.659321:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776569.659323:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776569.659325:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776569.659331:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000001:3.0:1775776569.659335:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776569.659341:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73100 x1862022710424192/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776569.659347:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776569.659350:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776569.659353:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f67e6552] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73100 x1862022710424192/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776569.659367:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776569.659369:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776569.659371:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776569.659372:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776569.659373:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776569.659375:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776569.659377:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13452 00000400:00000200:3.0:1775776569.659379:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776569.659382:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776569.659385:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776569.659388:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776569.659391:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776569.659393:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776569.659395:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:1.0:1775776569.659397:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1775776569.659398:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775776569.659401:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1775776569.659401:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776569.659402:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775776569.659404:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1775776569.659406:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776569.659409:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:1.0:1775776569.659412:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73100 x1862022710424192/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775776569.659412:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776569.659420:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776569.659423:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776569.659424:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776569.659425:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776569.659428:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776569.659429:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776569.659432:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776569.659432:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:1.0:1775776569.659437:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776569.659438:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00100000:1.0:1775776569.659441:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710424192:192.168.201.115@tcp:250:kworker.0 00000100:00080000:3.0:1775776569.659444:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776569.659447:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1775776569.659448:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776569.659450:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776569.659450:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776569.659452:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776569.659454:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776569.659455:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1775776569.659457:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776569.659459:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000010:1.0:1775776569.659460:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776569.659462:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775776569.659463:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776569.659465:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776569.659468:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776569.659469:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000010:3.0:1775776569.659473:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000400:00000010:1.0:1775776569.659475:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000020:00000040:3.0:1775776569.659475:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776569.659480:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776569.659481:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776569.659482:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776569.659483:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1775776569.659484:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 02000000:00000001:3.0:1775776569.659484:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776569.659488:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:3.0:1775776569.659491:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776569.659492:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710424192, portal 25 00000100:00000001:1.0:1775776569.659496:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00100000:3.0:1775776569.659496:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710424320/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776569.659499:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849016576 : -123823860535040 : ffff8f6202c73100) 00000100:00000001:3.0:1775776569.659505:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776569.659509:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776569.659510:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73100 x1862022710424192/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776594 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776569.659513:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710424320/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.659532:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775776569.659533:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776569.659536:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1775776569.659540:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00080000:3.0:1775776569.659541:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776569.659543:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776569.659544:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:1.0:1775776569.659545:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000001:3.0:1775776569.659546:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776569.659550:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13452 00000100:00000200:1.0:1775776569.659551:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710424192, offset 0 00000400:00000200:3.0:1775776569.659555:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776569.659557:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776569.659558:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1775776569.659563:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776569.659563:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776569.659564:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776569.659567:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776569.659569:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776569.659570:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776569.659570:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776569.659571:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776569.659573:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000400:00000200:1.0:1775776569.659574:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000001:3.0:1775776569.659575:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776569.659578:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776569.659580:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776569.659581:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776569.659584:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000040:3.0:1775776569.659584:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776569.659588:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776569.659590:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776569.659592:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:3.0:1775776569.659593:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776569.659596:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776569.659597:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776569.659600:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 02000000:00000001:3.0:1775776569.659600:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776569.659604:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776569.659605:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775776569.659607:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000008:00100000:3.0:1775776569.659607:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776569.659611:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776569.659613:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:3.0:1775776569.659613:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776569.659616:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775776569.659619:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:1.0:1775776569.659620:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453866:453866:256:4294967295] 192.168.201.115@tcp LPNI seq info [453791:453791:8:4294967295] 02000000:00000001:3.0:1775776569.659622:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776569.659625:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776569.659628:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776569.659628:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776569.659631:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776569.659632:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776569.659635:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000400:00000200:1.0:1775776569.659639:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:3.0:1775776569.659639:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776569.659645:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710424448/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1775776569.659648:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776569.659654:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000100:00000001:3.0:1775776569.659654:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776569.659656:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776569.659660:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710424448/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1775776569.659665:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775776569.659668:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776569.659670:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000100:00000040:3.0:1775776569.659670:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776569.659675:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776569.659677:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775776569.659681:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776569.659682:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776569.659688:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13462) 00000800:00000200:1.1:1775776569.659688:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1775776569.659696:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776569.659698:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776569.659705:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776569.659710:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776569.659715:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776569.659717:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776569.659723:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776569.659727:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73100 x1862022710424192/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776594 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.659737:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776569.659739:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73100 x1862022710424192/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776594 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.659744:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776569.659745:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776569.659750:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776569.659753:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776569.659755:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308128518 00000100:00000010:1.0:1775776569.659758:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2420. 00000100:00000001:1.0:1775776569.659762:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776569.659774:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308128518 00000100:00000010:1.0:1775776569.659776:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2360. 00000100:00000001:1.0:1775776569.659778:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776569.659781:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776569.659782:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776569.659784:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776569.659786:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776569/real 1775776569] req@ffff8f6202c73100 x1862022710424192/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776569.659793:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776569.659794:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776569.659796:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776569.659798:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776569.659799:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776569.659801:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776569.659804:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73100 x1862022710424192/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776569.659810:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73100 x1862022710424192/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.659816:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776569.659818:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776569.659821:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776569.659822:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776569.659825:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73100 x1862022710424192/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.659831:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776569.659835:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73100 x1862022710424192/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.659842:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776569.659845:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776569.659847:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776569.659848:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776569.659850:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776569.659852:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776569.659855:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776569.659858:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776569.659860:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776569.659864:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73100 x1862022710424192/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776569.659871:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710424192:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776569.659874:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776569.659875:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776569.659877:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73100 x1862022710424192/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.659881:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776569.659883:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776569.659884:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776569.659886:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776569.659887:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776569.659889:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776569.659890:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776569.659891:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37800. 02000000:00000001:1.0:1775776569.659894:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776569.659895:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776569.659896:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73100. 00000100:00000001:1.0:1775776569.659898:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776569.659899:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776569.659900:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776569.659905:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776569.659906:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776569.659907:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776569.659908:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776569.659909:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776569.659911:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70e00 x1862022710424320/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.659915:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776569.659916:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776569.659917:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776569.659919:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776569.659920:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776569.659921:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710424320:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776569.659924:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776569.659925:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776569.659926:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776569.659927:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776569.659929:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776569.659930:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776569.659933:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776569.659934:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776569.659937:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776569.659939:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710424320, portal 10 00000100:00000001:1.0:1775776569.659940:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776569.659941:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849007616 : -123823860544000 : ffff8f6202c70e00) 00000100:00000040:1.0:1775776569.659945:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70e00 x1862022710424320/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776594 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.659949:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776569.659951:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776569.659952:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776569.659953:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710424320, offset 0 00000400:00000200:1.0:1775776569.659956:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776569.659958:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776569.659960:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776569.659962:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776569.659963:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776569.659965:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776569.659966:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776569.659968:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453867:453867:256:4294967295] 192.168.201.115@tcp LPNI seq info [453792:453792:8:4294967295] 00000400:00000200:1.0:1775776569.659971:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776569.659975:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776569.659977:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776569.659979:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776569.659981:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776569.659983:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775776569.659985:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776569.659987:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1775776569.659989:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776569.659991:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776569.659992:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776569.659994:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776569.659996:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776569.659997:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776569.659999:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776569.660001:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70e00 x1862022710424320/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776594 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.660005:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776569.660007:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70e00 x1862022710424320/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776594 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.660015:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776569.660015:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776569.660017:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776569.660020:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776569.660021:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776569.660023:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308124683 00000100:00000001:1.0:1775776569.660024:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308128518 00000100:00000001:1.0:1775776569.660026:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308124683 00000100:00000001:1.0:1775776569.660027:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308128518 00000100:00000001:1.0:1775776569.660028:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776569.660029:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776569.660030:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776569.660033:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776569/real 1775776569] req@ffff8f6202c70e00 x1862022710424320/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776569.660038:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776569.660039:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776569.660040:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776569.660042:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776569.660042:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776569.660044:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776569.660046:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70e00 x1862022710424320/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776569.660050:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70e00 x1862022710424320/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.660053:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776569.660054:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776569.660055:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776569.660055:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776569.660057:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70e00 x1862022710424320/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.660061:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776569.660063:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70e00 x1862022710424320/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.660072:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776569.660074:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776569.660075:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776569.660076:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776569.660077:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776569.660080:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776569.660081:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776569.660083:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776569.660085:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776569.660087:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70e00 x1862022710424320/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776569.660091:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710424320:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776569.660093:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776569.660095:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70000 x1862022710424448/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.660099:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776569.660100:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776569.660101:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776569.660102:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776569.660103:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776569.660104:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710424448:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776569.660106:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776569.660108:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776569.660109:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776569.660110:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776569.660112:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776569.660113:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776569.660115:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776569.660116:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776569.660118:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776569.660119:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710424448, portal 10 00000100:00000001:1.0:1775776569.660120:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776569.660121:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004032 : -123823860547584 : ffff8f6202c70000) 00000100:00000040:1.0:1775776569.660125:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70000 x1862022710424448/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776594 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.660128:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776569.660130:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776569.660131:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776569.660132:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710424448, offset 0 00000400:00000200:1.0:1775776569.660154:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776569.660158:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776569.660161:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776569.660164:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776569.660167:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776569.660170:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776569.660172:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776569.660176:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453868:453868:256:4294967295] 192.168.201.115@tcp LPNI seq info [453793:453793:8:4294967295] 00000400:00000200:1.0:1775776569.660179:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776569.660185:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776569.660189:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776569.660193:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776569.660197:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776569.660201:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1775776569.660205:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776569.660208:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1775776569.660211:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776569.660213:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776569.660230:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776569.660234:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776569.660237:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776569.660238:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776569.660241:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776569.660244:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70000 x1862022710424448/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776594 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.660252:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776569.660254:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70000 x1862022710424448/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776594 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.660260:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776569.660262:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776569.660265:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776569.660267:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776569.660271:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776569.660273:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308124684 00000100:00000001:1.0:1775776569.660274:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308128518 00000100:00000001:1.0:1775776569.660276:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308124684 00000100:00000001:1.0:1775776569.660277:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308128518 00000100:00000001:1.0:1775776569.660278:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776569.660279:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776569.660280:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776569.660283:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776569/real 1775776569] req@ffff8f6202c70000 x1862022710424448/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776569.660288:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776569.660288:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776569.660290:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776569.660291:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776569.660292:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776569.660294:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776569.660295:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70000 x1862022710424448/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776569.660300:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70000 x1862022710424448/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.660303:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776569.660304:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776569.660305:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776569.660306:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776569.660307:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70000 x1862022710424448/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.660311:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776569.660313:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70000 x1862022710424448/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.660317:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776569.660318:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776569.660319:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776569.660320:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776569.660321:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776569.660323:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776569.660325:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776569.660327:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776569.660329:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776569.660333:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70000 x1862022710424448/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776569.660340:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710424448:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776569.660343:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776569.660345:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776569.660348:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70e00 x1862022710424320/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.660354:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776569.660356:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776569.660357:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776569.660360:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776569.660361:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776569.660362:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776569.660364:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776569.660366:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37c00. 02000000:00000001:1.0:1775776569.660368:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776569.660370:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776569.660371:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70e00. 00000100:00000001:1.0:1775776569.660373:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776569.660374:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776569.660376:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776569.660379:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70000 x1862022710424448/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776594 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776569.660385:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776569.660386:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776569.660388:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776569.660390:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776569.660391:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776569.660392:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776569.660394:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776569.660395:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35000. 02000000:00000001:1.0:1775776569.660397:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776569.660398:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776569.660399:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70000. 00000100:00000001:1.0:1775776569.660401:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776569.660402:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776569.660404:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776569.660406:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776569.660407:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776569.660408:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776569.660410:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776569.787236:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776569.787240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776569.787244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776569.787245:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776569.787248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776569.787249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776569.787307:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776569.787312:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776569.787315:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776570.683243:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776570.683249:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776570.683250:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776570.683254:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776570.683256:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776570.683259:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776570.683261:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776570.683263:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776570.683264:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776570.811260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776570.811266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776570.811270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776570.811272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776570.811275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776570.811277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776570.811331:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776570.811334:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776570.811336:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776571.707258:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776571.707263:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776571.707264:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776571.707268:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776571.707271:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776571.707274:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776571.707276:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776571.707278:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776571.707280:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776571.835250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776571.835255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776571.835259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776571.835262:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776571.835264:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776571.835268:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776571.835270:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776571.835273:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776571.835274:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776571.835276:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776571.835280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776571.835282:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776571.835283:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776571.835284:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776571.835286:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776571.835287:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776571.835289:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776571.835290:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776571.835348:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776571.835351:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776571.835354:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776572.731228:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776572.731233:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776572.731235:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776572.731241:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776572.731244:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776572.731247:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776572.731250:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776572.731252:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776572.731254:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776572.859305:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776572.859311:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776572.859316:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776572.859317:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776572.859321:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776572.859322:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776572.859380:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776572.859383:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776572.859385:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776573.563293:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308128518, at 4308128576 00000100:00000040:1.0:1775776573.563294:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308128518, at 4308128576 00000100:00000010:0.0:1775776573.563318:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2420. 00000100:00000010:1.0:1775776573.563321:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2360. 00000100:00000001:1.0:1775776573.755327:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776573.755334:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776573.755336:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776573.755341:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776573.755345:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776573.755349:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776573.755352:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776573.755354:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776573.755357:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776573.883268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776573.883275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776573.883282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776573.883283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776573.883288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776573.883289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776573.883346:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776573.883351:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776573.883355:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776574.779301:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776574.779315:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776574.779319:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776574.779323:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776574.779329:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776574.779333:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1775776574.779338:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13457 00000100:00000001:1.0:1775776574.779338:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776574.779342:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776574.779346:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776574.779348:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776574.779351:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775776574.779352:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776574.779354:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776574.779357:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776574.779360:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776574.779363:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776574.779367:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776574.779371:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776574.779374:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776574.779377:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776574.779380:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776574.779385:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776574.779387:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776574.779390:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776574.779394:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776574.779397:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776574.779399:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776574.779402:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776574.779404:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776574.779407:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776574.779409:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776574.779412:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776574.779415:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776574.779418:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776574.779422:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776574.779426:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776574.779429:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776574.779432:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776574.779434:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776574.779446:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062d80. 00000020:00000040:0.0:1775776574.779450:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776574.779454:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776574.779456:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776574.779460:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776574.779463:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776574.779468:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776574.779477:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6000. 00000100:00000001:0.0:1775776574.779485:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776574.779495:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062d80 x1862022710424576/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776574.779506:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776574.779511:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776574.779516:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009965f80b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062d80 x1862022710424576/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776574.779539:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776574.779542:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776574.779546:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776574.779548:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776574.779549:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776574.779552:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776574.779555:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13457 00000400:00000200:0.0:1775776574.779559:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776574.779563:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:1.0:1775776574.779566:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1775776574.779567:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1775776574.779570:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1775776574.779573:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776574.779573:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1775776574.779577:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776574.779581:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e062d80 x1862022710424576/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776574.779582:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776574.779588:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776574.779593:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775776574.779593:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:0.0:1775776574.779596:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:1.0:1775776574.779596:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776574.779600:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:1.0:1775776574.779601:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:0.0:1775776574.779603:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 02000000:00000001:1.0:1775776574.779604:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1775776574.779608:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:1.0:1775776574.779608:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776574.779613:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00100000:1.0:1775776574.779613:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e062d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710424576:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776574.779618:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1775776574.779621:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776574.779623:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1775776574.779624:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1775776574.779626:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776574.779629:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:1.0:1775776574.779630:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1775776574.779633:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000010:1.0:1775776574.779636:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4fc00. 00000020:00000040:0.0:1775776574.779637:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:1.0:1775776574.779639:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776574.779643:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000400:00000200:1.0:1775776574.779646:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776574.779648:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776574.779650:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 02000000:00000001:0.0:1775776574.779651:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:1.0:1775776574.779658:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 02000000:00000001:0.0:1775776574.779659:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776574.779663:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776574.779665:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000200:1.0:1775776574.779667:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710424576, portal 25 00000008:00100000:0.0:1775776574.779669:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775776574.779670:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776574.779672:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574564736 : -123823134986880 : ffff8f622e062d80) 00000008:00000001:0.0:1775776574.779676:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776574.779679:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e062d80 x1862022710424576/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776599 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776574.779680:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776574.779687:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e062a00. 00000100:00000001:1.0:1775776574.779687:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776574.779691:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000020:00000040:0.0:1775776574.779692:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000400:00000010:1.0:1775776574.779694:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 02000000:00000001:0.0:1775776574.779696:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000200:1.0:1775776574.779697:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710424576, offset 0 02000000:00000001:0.0:1775776574.779699:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776574.779702:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1775776574.779703:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 02000000:00000001:0.0:1775776574.779705:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776574.779707:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776574.779712:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5800. 00000400:00000200:1.0:1775776574.779712:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776574.779718:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776574.779719:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776574.779725:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00100000:0.0:1775776574.779727:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e062a00 x1862022710424704/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776574.779730:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776574.779736:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776574.779740:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:0.0:1775776574.779743:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775776574.779746:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453869:453869:256:4294967295] 192.168.201.115@tcp LPNI seq info [453794:453794:8:4294967295] 00000100:00000001:0.0:1775776574.779748:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776574.779754:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:0.0:1775776574.779757:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009753f9f6] to pc [ptlrpcd_rcv+-1] req@ffff8f622e062a00 x1862022710424704/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776574.779768:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776574.779778:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:0.0:1775776574.779779:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775776574.779782:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000100:00080000:0.0:1775776574.779783:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776574.779788:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776574.779792:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:1.0:1775776574.779792:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00080000:0.0:1775776574.779795:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.0:1775776574.779796:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000100:00000001:0.0:1775776574.779800:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000800:00000200:1.1:1775776574.779803:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00080000:0.0:1775776574.779806:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13457 00000800:00000200:1.1:1775776574.779807:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1775776574.779814:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776574.779816:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:0.0:1775776574.779817:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776574.779823:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000400:00000200:1.0:1775776574.779823:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:0.0:1775776574.779828:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1775776574.779830:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000040:0.0:1775776574.779835:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776574.779836:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000100:00000001:0.0:1775776574.779840:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1775776574.779840:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000040:0.0:1775776574.779844:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776574.779847:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:1.0:1775776574.779848:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1775776574.779852:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000200:1.0:1775776574.779852:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e062d80 x1862022710424576/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776599 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775776574.779855:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776574.779858:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776574.779862:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000100:00000001:1.0:1775776574.779863:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1775776574.779866:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:1.0:1775776574.779868:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e062d80 x1862022710424576/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776599 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1775776574.779871:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000001:1.0:1775776574.779878:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776574.779880:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776574.779880:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776574.779884:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776574.779886:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776574.779887:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776574.779891:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776574.779891:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776574.779894:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308133638 00000020:00000040:0.0:1775776574.779896:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00000010:1.0:1775776574.779899:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2360. 00000100:00080000:0.0:1775776574.779901:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776574.779905:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776574.779906:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1775776574.779908:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776574.779912:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776574.779914:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308133638 02000000:00000001:0.0:1775776574.779915:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776574.779918:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e22a0. 00000100:00000001:0.0:1775776574.779919:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776574.779921:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000008:00100000:0.0:1775776574.779922:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776574.779926:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776574.779926:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776574.779928:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776574.779929:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776574.779931:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000010:0.0:1775776574.779936:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061500. 00000100:00100000:1.0:1775776574.779936:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776574/real 1775776574] req@ffff8f622e062d80 x1862022710424576/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775776574.779941:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776574.779947:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000001:1.0:1775776574.779948:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1775776574.779949:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1775776574.779950:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776574.779952:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000100:00000001:0.0:1775776574.779953:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1775776574.779954:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 02000000:00000001:0.0:1775776574.779956:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1775776574.779957:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 02000000:00000001:0.0:1775776574.779960:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776574.779960:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776574.779964:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e062d80 x1862022710424576/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1775776574.779965:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6c00. 00000100:00000001:0.0:1775776574.779971:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776574.779974:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e062d80 x1862022710424576/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776574.779980:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061500 x1862022710424832/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776574.779981:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776574.779983:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776574.779986:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776574.779989:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776574.779995:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e062d80 x1862022710424576/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776574.780016:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776574.780019:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1775776574.780021:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e062d80 x1862022710424576/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776574.780024:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776574.780030:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:0.0:1775776574.780032:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006cb8d992] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061500 x1862022710424832/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776574.780033:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776574.780042:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776574.780043:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:0.0:1775776574.780045:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776574.780049:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776574.780050:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776574.780053:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000040:0.0:1775776574.780055:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776574.780056:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:0.0:1775776574.780062:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00080000:1.0:1775776574.780063:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:0.0:1775776574.780070:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13467) 00000100:00080000:1.0:1775776574.780071:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776574.780076:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776574.780081:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e062d80 x1862022710424576/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776574.780090:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e062d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710424576:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776574.780095:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776574.780097:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776574.780099:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e062d80 x1862022710424576/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776574.780106:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776574.780108:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776574.780111:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776574.780114:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776574.780116:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776574.780118:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776574.780120:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776574.780122:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:1.0:1775776574.780169:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776574.780172:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776574.780174:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e062d80. 00000100:00000001:1.0:1775776574.780178:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776574.780180:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776574.780182:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776574.780189:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776574.780191:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776574.780193:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776574.780194:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776574.780195:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776574.780199:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e062a00 x1862022710424704/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776574.780205:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776574.780207:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776574.780209:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776574.780210:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776574.780212:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776574.780214:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e062a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710424704:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776574.780217:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776574.780219:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776574.780220:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776574.780222:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776574.780225:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776574.780226:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776574.780230:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776574.780232:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776574.780236:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776574.780238:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710424704, portal 10 00000100:00000001:1.0:1775776574.780240:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776574.780241:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574563840 : -123823134987776 : ffff8f622e062a00) 00000100:00000040:1.0:1775776574.780248:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e062a00 x1862022710424704/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776599 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776574.780254:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776574.780256:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776574.780258:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776574.780260:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710424704, offset 0 00000400:00000200:1.0:1775776574.780263:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776574.780266:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776574.780270:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776574.780272:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776574.780275:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776574.780277:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776574.780279:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776574.780282:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453870:453870:256:4294967295] 192.168.201.115@tcp LPNI seq info [453795:453795:8:4294967295] 00000400:00000200:1.0:1775776574.780286:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776574.780291:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776574.780295:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776574.780298:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776574.780301:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776574.780304:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775776574.780308:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776574.780310:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1775776574.780313:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776574.780314:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776574.780317:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776574.780320:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776574.780322:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776574.780324:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776574.780326:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776574.780329:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e062a00 x1862022710424704/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776599 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776574.780337:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776574.780340:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e062a00 x1862022710424704/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776599 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776574.780351:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776574.780352:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776574.780355:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776574.780358:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776574.780361:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776574.780363:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308129804 00000100:00000001:1.0:1775776574.780366:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308133638 00000100:00000001:1.0:1775776574.780368:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308129804 00000100:00000001:1.0:1775776574.780370:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308133638 00000100:00000001:1.0:1775776574.780372:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776574.780373:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776574.780375:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776574.780378:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776574/real 1775776574] req@ffff8f622e062a00 x1862022710424704/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776574.780384:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776574.780386:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776574.780387:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776574.780389:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776574.780391:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776574.780393:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776574.780395:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e062a00 x1862022710424704/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776574.780403:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e062a00 x1862022710424704/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776574.780409:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776574.780410:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776574.780411:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776574.780413:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776574.780415:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e062a00 x1862022710424704/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776574.780421:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776574.780425:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e062a00 x1862022710424704/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776574.780431:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776574.780433:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776574.780434:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776574.780436:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776574.780437:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776574.780442:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776574.780445:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776574.780448:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776574.780450:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776574.780453:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e062a00 x1862022710424704/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776574.780460:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e062a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710424704:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776574.780463:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776574.780466:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061500 x1862022710424832/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776574.780472:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776574.780473:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776574.780474:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776574.780476:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776574.780478:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776574.780480:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710424832:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776574.780483:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776574.780485:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776574.780487:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776574.780488:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776574.780491:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776574.780493:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776574.780496:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776574.780498:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776574.780501:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776574.780503:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710424832, portal 10 00000100:00000001:1.0:1775776574.780504:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776574.780506:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574558464 : -123823134993152 : ffff8f622e061500) 00000100:00000040:1.0:1775776574.780510:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061500 x1862022710424832/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776599 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776574.780515:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776574.780517:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776574.780519:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000100:00000200:1.0:1775776574.780521:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710424832, offset 0 00000400:00000200:1.0:1775776574.780525:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776574.780528:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776574.780531:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776574.780534:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776574.780536:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776574.780538:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776574.780540:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776574.780543:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453871:453871:256:4294967295] 192.168.201.115@tcp LPNI seq info [453796:453796:8:4294967295] 00000400:00000200:1.0:1775776574.780547:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776574.780552:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776574.780557:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776574.780561:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000800:00000200:1.0:1775776574.780563:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776574.780565:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1775776574.780567:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776574.780569:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1775776574.780571:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776574.780572:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:1.0:1775776574.780574:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776574.780576:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776574.780578:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:1.0:1775776574.780579:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:1.0:1775776574.780581:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776574.780584:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061500 x1862022710424832/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776599 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776574.780589:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776574.780590:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061500 x1862022710424832/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776599 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776574.780594:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776574.780595:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776574.780597:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776574.780599:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776574.780601:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776574.780602:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308129804 00000100:00000001:1.0:1775776574.780604:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308133638 00000100:00000001:1.0:1775776574.780605:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308129804 00000100:00000001:1.0:1775776574.780607:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308133638 00000100:00000001:1.0:1775776574.780609:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776574.780611:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776574.780613:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776574.780616:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776574/real 1775776574] req@ffff8f622e061500 x1862022710424832/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776574.780623:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776574.780624:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776574.780626:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776574.780628:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776574.780629:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776574.780632:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776574.780635:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061500 x1862022710424832/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776574.780643:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061500 x1862022710424832/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776574.780650:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776574.780651:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776574.780653:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776574.780654:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776574.780657:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061500 x1862022710424832/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776574.780664:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776574.780667:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061500 x1862022710424832/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776574.780671:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776574.780672:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776574.780673:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776574.780674:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776574.780675:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776574.780677:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776574.780678:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776574.780680:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776574.780681:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776574.780683:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061500 x1862022710424832/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776574.780687:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710424832:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776574.780689:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776574.780690:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776574.780692:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e062a00 x1862022710424704/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776574.780696:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776574.780697:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776574.780698:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776574.780700:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776574.780700:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776574.780701:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776574.780703:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776574.780704:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:1.0:1775776574.780705:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776574.780706:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776574.780707:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e062a00. 00000100:00000001:1.0:1775776574.780709:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776574.780709:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776574.780710:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776574.780712:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061500 x1862022710424832/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776599 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776574.780716:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776574.780716:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776574.780717:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776574.780718:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776574.780719:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776574.780719:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776574.780720:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776574.780721:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e6c00. 02000000:00000001:1.0:1775776574.780722:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776574.780722:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776574.780723:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061500. 00000100:00000001:1.0:1775776574.780724:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776574.780725:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776574.780726:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776574.780728:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776574.780729:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776574.780730:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776574.780731:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776574.907284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776574.907291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776574.907297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776574.907299:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776574.907303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776574.907304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776574.907370:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776574.907375:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776574.907379:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776575.803436:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776575.803445:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776575.803447:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776575.803454:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776575.803459:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776575.803473:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776575.803476:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776575.803479:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776575.803482:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776575.932302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776575.932310:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776575.932318:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776575.932319:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776575.932324:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776575.932325:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776575.932380:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776575.932385:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776575.932389:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776576.827237:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776576.827246:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776576.827248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776576.827254:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776576.827258:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776576.827262:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776576.827265:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776576.827268:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776576.827271:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776576.955300:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776576.955310:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776576.955312:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776576.955317:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776576.955323:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776576.955326:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776576.955328:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776576.955330:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776576.955334:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776576.955336:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776576.955341:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776576.955342:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776576.955404:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776576.955409:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776576.955421:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776577.851274:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776577.851283:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776577.851285:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776577.851293:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776577.851297:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776577.851310:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776577.851314:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776577.851320:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776577.851323:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776577.979798:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776577.979806:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776577.979813:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776577.979814:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776577.979817:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776577.979818:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776577.979942:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776577.979947:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776577.979951:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776578.683342:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308133638, at 4308133696 00000100:00000010:1.0:1775776578.683366:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e22a0. 00000100:00000040:0.0:1775776578.683626:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308133638, at 4308133696 00000100:00000010:0.0:1775776578.683635:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2360. 00000100:00000001:1.0:1775776578.875320:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776578.875331:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776578.875334:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776578.875343:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776578.875348:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776578.875354:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776578.875358:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776578.875362:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776578.875365:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776579.003252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776579.003259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776579.003266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776579.003267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776579.003272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776579.003273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776579.003332:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776579.003335:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776579.003338:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776579.899240:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776579.899250:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1775776579.899251:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776579.899253:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776579.899262:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1775776579.899263:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776579.899265:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776579.899269:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776579.899271:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776579.899274:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776579.899277:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776579.899281:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776579.899283:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776579.899285:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776579.899293:0:191261:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13462 00000400:00000200:3.0:1775776579.899305:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776579.899313:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776579.899316:0:191261:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776579.899320:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776579.899324:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776579.899327:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776579.899330:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776579.899334:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776579.899336:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776579.899340:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776579.899343:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776579.899345:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776579.899347:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776579.899351:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776579.899353:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776579.899355:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776579.899357:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776579.899360:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776579.899363:0:191261:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776579.899365:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776579.899370:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776579.899373:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776579.899376:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776579.899379:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776579.899382:0:191261:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776579.899393:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000020:00000040:3.0:1775776579.899397:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776579.899408:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776579.899410:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776579.899413:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776579.899420:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776579.899424:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776579.899434:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000100:00000001:3.0:1775776579.899442:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776579.899452:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710424960/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776579.899464:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776579.899468:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776579.899473:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710424960/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776579.899493:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776579.899497:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776579.899500:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776579.899502:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776579.899503:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776579.899506:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776579.899509:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13462 00000400:00000200:3.0:1775776579.899513:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776579.899522:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776579.899525:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776579.899528:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776579.899530:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776579.899532:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1775776579.899532:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776579.899536:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:1.0:1775776579.899537:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1775776579.899541:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775776579.899542:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1775776579.899543:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776579.899546:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776579.899548:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776579.899551:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:1.0:1775776579.899552:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71c00 x1862022710424960/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775776579.899554:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776579.899562:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776579.899566:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776579.899570:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1775776579.899570:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776579.899573:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776579.899574:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776579.899579:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:1.0:1775776579.899581:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1775776579.899582:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000001:1.0:1775776579.899585:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776579.899585:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776579.899589:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776579.899590:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776579.899595:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710424960:192.168.201.115@tcp:250:kworker.0 02000000:00000001:3.0:1775776579.899596:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776579.899600:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776579.899605:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776579.899606:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00100000:3.0:1775776579.899610:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775776579.899613:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:3.0:1775776579.899614:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776579.899617:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776579.899618:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776579.899623:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 02000000:00000001:1.0:1775776579.899624:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:3.0:1775776579.899628:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000010:1.0:1775776579.899631:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:3.0:1775776579.899631:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776579.899632:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776579.899636:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776579.899636:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776579.899639:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776579.899641:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776579.899645:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000010:3.0:1775776579.899646:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000400:00000010:1.0:1775776579.899650:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000001:3.0:1775776579.899651:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776579.899658:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710425088/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1775776579.899660:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000001:3.0:1775776579.899670:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776579.899672:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776579.899673:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710424960, portal 25 00000100:00000040:3.0:1775776579.899677:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710425088/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776579.899678:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776579.899681:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849011200 : -123823860540416 : ffff8f6202c71c00) 00000100:00000001:3.0:1775776579.899692:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776579.899694:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71c00 x1862022710424960/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776604 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775776579.899695:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776579.899702:0:191261:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776579.899706:0:191261:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776579.899709:0:191261:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776579.899711:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775776579.899712:0:191261:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776579.899717:0:191261:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13462 00000100:00000040:1.0:1775776579.899721:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000200:3.0:1775776579.899724:0:191261:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776579.899726:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f35d8. 00000400:00000200:3.0:1775776579.899729:0:191261:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000200:1.0:1775776579.899731:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710424960, offset 0 00000100:00080000:3.0:1775776579.899732:0:191261:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776579.899738:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776579.899740:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776579.899745:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776579.899745:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776579.899749:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776579.899751:0:191261:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776579.899753:0:191261:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776579.899754:0:191261:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776579.899756:0:191261:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000400:00000200:1.0:1775776579.899760:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776579.899767:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000020:00000001:3.0:1775776579.899768:0:191261:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776579.899772:0:191261:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776579.899776:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1775776579.899777:0:191261:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776579.899781:0:191261:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776579.899785:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000040:3.0:1775776579.899785:0:191261:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776579.899790:0:191261:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776579.899792:0:191261:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776579.899794:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00080000:3.0:1775776579.899798:0:191261:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000400:00000200:1.0:1775776579.899801:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:3.0:1775776579.899804:0:191261:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776579.899807:0:191261:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776579.899811:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453872:453872:256:4294967295] 192.168.201.115@tcp LPNI seq info [453797:453797:8:4294967295] 02000000:00000001:3.0:1775776579.899811:0:191261:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776579.899814:0:191261:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776579.899818:0:191261:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776579.899821:0:191261:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1775776579.899822:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000008:00000001:3.0:1775776579.899827:0:191261:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776579.899829:0:191261:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776579.899833:0:191261:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775776579.899837:0:191261:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:1.0:1775776579.899842:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 02000000:00000001:3.0:1775776579.899843:0:191261:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776579.899845:0:191261:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776579.899849:0:191261:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776579.899851:0:191261:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776579.899853:0:191261:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776579.899854:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 02000000:00000010:3.0:1775776579.899858:0:191261:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000100:00000001:3.0:1775776579.899861:0:191261:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775776579.899862:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230300. 00000100:00100000:3.0:1775776579.899867:0:191261:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710425216/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1775776579.899876:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775776579.899879:0:191261:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776579.899882:0:191261:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776579.899886:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776579.899895:0:191261:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710425216/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776579.899904:0:191261:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776579.899907:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776579.899910:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776579.899911:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776579.899916:0:191261:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776579.899919:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775776579.899922:0:191261:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776579.899924:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308138758 00000100:00000040:3.0:1775776579.899928:0:191261:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13472) 00000100:00000010:1.0:1775776579.899931:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e22a0. 00000100:00000001:1.0:1775776579.899940:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776579.899948:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308138758 00000100:00000010:1.0:1775776579.899953:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e29c0. 00000100:00000001:1.0:1775776579.899956:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776579.899960:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776579.899964:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776579.899966:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776579.899969:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776579.899977:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776579.899978:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775776579.899980:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776579.899982:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776579.899983:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776579.899988:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73480 x1862022710425088/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776579.899995:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776579.899997:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776579.899999:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776579.900001:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776579.900003:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776579.900006:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710425088:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776579.900009:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776579.900011:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776579.900012:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776579.900014:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776579.900017:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776579.900020:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776579.900023:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776579.900026:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec60. 00000400:00000010:1.0:1775776579.900030:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3660. 00000100:00000200:1.0:1775776579.900033:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710425088, portal 10 00000100:00000001:1.0:1775776579.900035:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776579.900036:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849017472 : -123823860534144 : ffff8f6202c73480) 00000100:00000040:1.0:1775776579.900060:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73480 x1862022710425088/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776604 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776579.900067:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776579.900070:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776579.900073:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3ee0. 00000100:00000200:1.0:1775776579.900075:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710425088, offset 0 00000400:00000200:1.0:1775776579.900080:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776579.900084:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776579.900087:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776579.900090:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776579.900093:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776579.900095:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776579.900097:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776579.900100:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453873:453873:255:4294967295] 192.168.201.115@tcp LPNI seq info [453798:453798:7:4294967295] 00000400:00000200:1.0:1775776579.900105:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776579.900111:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776579.900116:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776579.900119:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230a00. 00000800:00000200:1.0:1775776579.900123:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776579.900127:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:1.0:1775776579.900129:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776579.900151:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776579.900154:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776579.900157:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308134924 00000100:00000001:1.0:1775776579.900159:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308138758 00000100:00000001:1.0:1775776579.900161:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308134924 00000100:00000001:1.0:1775776579.900164:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308138758 00000100:00000001:1.0:1775776579.900166:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776579.900168:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776579.900170:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776579.900174:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70380 x1862022710425216/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776579.900182:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776579.900183:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776579.900186:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776579.900187:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776579.900190:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776579.900192:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710425216:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776579.900195:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776579.900197:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776579.900198:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776579.900200:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776579.900204:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776579.900205:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776579.900208:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776579.900210:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fe000. 00000400:00000010:1.0:1775776579.900213:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776579.900216:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710425216, portal 10 00000100:00000001:1.0:1775776579.900218:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776579.900219:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004928 : -123823860546688 : ffff8f6202c70380) 00000100:00000040:1.0:1775776579.900224:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70380 x1862022710425216/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776604 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776579.900230:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776579.900232:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776579.900235:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3880. 00000100:00000200:1.0:1775776579.900237:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710425216, offset 0 00000400:00000200:1.0:1775776579.900241:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776579.900245:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776579.900248:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776579.900250:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776579.900253:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776579.900255:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776579.900257:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776579.900259:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453874:453874:254:4294967295] 192.168.201.115@tcp LPNI seq info [453799:453799:6:4294967295] 00000400:00000200:1.0:1775776579.900263:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776579.900270:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776579.900274:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776579.900277:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230700. 00000800:00000200:1.0:1775776579.900281:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776579.900284:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:1.0:1775776579.900287:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776579.900289:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776579.900292:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776579.900294:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308134924 00000100:00000001:1.0:1775776579.900296:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308138758 00000100:00000001:1.0:1775776579.900298:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308134924 00000100:00000001:1.0:1775776579.900300:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308138758 00000100:00000001:1.0:1775776579.900302:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776579.900303:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776579.900306:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776579.900307:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776579.900310:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776579.900312:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1775776579.900314:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776579.900315:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776579.900316:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776579.900318:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776580.027227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776580.027234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776580.027241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776580.027242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776580.027246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776580.027247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776580.027302:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776580.027306:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776580.027310:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1775776580.923255:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1775776580.923261:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776580.923263:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776580.923269:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:1.0:1775776580.923273:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776580.923276:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776580.923279:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776580.923281:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776580.923284:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776581.051294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776581.051303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776581.051310:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776581.051312:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776581.051317:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776581.051318:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776581.051378:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776581.051381:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776581.051385:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776581.947314:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776581.947322:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776581.947324:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776581.947328:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776581.947332:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776581.947335:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776581.947338:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776581.947340:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776581.947343:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776582.075248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776582.075252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776582.075257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776582.075261:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776582.075263:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776582.075266:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776582.075269:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776582.075271:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776582.075273:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776582.075276:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776582.075280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776582.075281:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776582.075282:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776582.075284:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776582.075285:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776582.075286:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776582.075288:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776582.075290:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776582.075345:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776582.075348:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776582.075351:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776582.971389:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776582.971396:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776582.971398:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776582.971404:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776582.971407:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776582.971410:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776582.971412:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776582.971414:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776582.971416:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:3.0:1775776582.971519:0:2402:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:3.0:1775776582.971557:0:2402:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1775776582.971567:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776582.971571:0:2402:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:3.0:1775776582.971576:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776582.971580:0:2402:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:3.0:1775776582.971586:0:2402:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775776582.971590:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230300. 00000400:00000200:3.0:1775776582.971594:0:2402:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775776582.971599:0:2402:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775776582.971605:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f35d8 00000400:00000010:3.0:1775776582.971607:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f35d8. 00000100:00000001:3.0:1775776582.971613:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776582.971621:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c71c00 x1862022710424960/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776604 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776582.971644:0:2402:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776582.971648:0:2402:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71c00 x1862022710424960/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776604 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776582.971656:0:2402:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776582.971658:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1775776582.971667:0:2402:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775776582.971670:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230a00. 00000400:00000200:3.0:1775776582.971673:0:2402:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1775776582.971677:0:2402:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775776582.971681:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3ee0 00000400:00000010:3.0:1775776582.971683:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3ee0. 00000100:00000001:3.0:1775776582.971686:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776582.971691:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c73480 x1862022710425088/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776604 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776582.971700:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776582.971700:0:2402:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1775776582.971704:0:2402:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73480 x1862022710425088/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776604 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776582.971711:0:2402:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776582.971713:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776582.971715:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776582.971719:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000800:00000100:3.0:1775776582.971720:0:2402:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:3.0:1775776582.971724:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230700. 00000400:00000200:3.0:1775776582.971727:0:2402:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00100000:1.0:1775776582.971729:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776579/real 1775776582] req@ffff8f6202c73480 x1862022710425088/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1775776582.971733:0:2402:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1775776582.971738:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3880 00000400:00000010:3.0:1775776582.971742:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3880. 00000100:00000001:3.0:1775776582.971746:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1775776582.971753:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6202c70380 x1862022710425216/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776604 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776582.971758:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776582.971761:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1775776582.971764:0:2402:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000400:00000010:1.0:1775776582.971766:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec60. 00000100:00000040:3.0:1775776582.971767:0:2402:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70380 x1862022710425216/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776604 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776582.971772:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3660 00000400:00000010:1.0:1775776582.971776:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3660. 00000100:00000001:3.0:1775776582.971777:0:2402:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776582.971780:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776582.971781:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776582.971788:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73480 x1862022710425088/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776582.971797:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73480 x1862022710425088/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776582.971803:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776582.971805:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776582.971807:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776582.971809:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776582.971812:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73480 x1862022710425088/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776582.971818:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776582.971823:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73480 x1862022710425088/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776582.971831:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776582.971833:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776582.971835:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776582.971841:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776582.971843:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776582.971849:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776582.971852:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776582.971856:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776582.971858:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776582.971862:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73480 x1862022710425088/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776582.971869:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710425088:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776582.971873:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776582.971877:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776579/real 1775776582] req@ffff8f6202c70380 x1862022710425216/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776582.971883:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776582.971884:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776582.971885:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fe000. 00000400:00000200:1.0:1775776582.971888:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776582.971889:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776582.971891:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776582.971893:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70380 x1862022710425216/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776582.971904:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70380 x1862022710425216/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776582.971909:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776582.971910:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776582.971911:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776582.971913:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776582.971916:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70380 x1862022710425216/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776582.971921:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776582.971924:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70380 x1862022710425216/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776582.971930:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776582.971932:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776582.971934:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776582.971935:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776582.971937:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776582.971938:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776582.971940:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776582.971944:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776582.971946:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776582.971949:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70380 x1862022710425216/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776582.971956:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710425216:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776582.971959:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776582.971963:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776579/real 1775776582] req@ffff8f6202c71c00 x1862022710424960/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776582.971970:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776582.971971:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776582.971973:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776582.971975:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776582.971977:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776582.971979:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776582.971982:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71c00 x1862022710424960/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776582.972018:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71c00 x1862022710424960/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776582.972024:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776582.972026:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776582.972028:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776582.972029:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776582.972032:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71c00 x1862022710424960/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776582.972043:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776582.972047:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71c00 x1862022710424960/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776582.972052:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776582.972054:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776582.972055:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776582.972057:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776582.972058:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776582.972059:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776582.972061:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776582.972065:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776582.972067:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776582.972072:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71c00 x1862022710424960/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776582.972079:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710424960:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776582.972083:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776582.972085:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776582.972087:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73480 x1862022710425088/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776582.972094:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776582.972097:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776582.972101:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4fc00. 02000000:00000001:1.0:1775776582.972105:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776582.972108:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776582.972109:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776582.972111:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776582.972114:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36c00. 02000000:00000001:1.0:1775776582.972117:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776582.972120:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776582.972122:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73480. 00000100:00000001:1.0:1775776582.972125:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776582.972127:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776582.972128:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776582.972131:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70380 x1862022710425216/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776582.972184:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776582.972186:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776582.972188:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776582.972191:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776582.972192:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776582.972194:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776582.972195:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776582.972197:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35c00. 02000000:00000001:1.0:1775776582.972200:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776582.972201:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776582.972203:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70380. 00000100:00000001:1.0:1775776582.972205:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776582.972207:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776582.972209:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776582.972212:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71c00 x1862022710424960/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776604 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776582.972218:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776582.972220:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776582.972222:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776582.972225:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776582.972226:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776582.972228:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776582.972229:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776582.972230:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34000. 02000000:00000001:1.0:1775776582.972232:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776582.972234:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776582.972235:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71c00. 00000100:00000001:1.0:1775776582.972239:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776582.972240:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776582.972243:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776582.972249:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776582.972251:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776582.972253:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776582.972255:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776583.099223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776583.099227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776583.099230:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776583.099231:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776583.099234:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776583.099234:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776583.099294:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776583.099298:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776583.099301:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776583.803252:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308138758, at 4308138816 00000100:00000040:1.0:1775776583.803252:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308138758, at 4308138816 00000100:00000010:0.0:1775776583.803269:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e22a0. 00000100:00000010:1.0:1775776583.803279:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e29c0. 00000100:00000001:0.0:1775776583.995293:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776583.995299:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776583.995300:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776583.995304:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776583.995306:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776583.995309:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776583.995311:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776583.995313:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776583.995315:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776584.123262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776584.123267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776584.123273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776584.123274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776584.123278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776584.123279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776584.123341:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776584.123344:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776584.123346:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776585.019265:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776585.019271:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776585.019272:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776585.019276:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776585.019279:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776585.019281:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776585.019283:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776585.019285:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776585.019287:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776585.019305:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776585.019309:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776585.019311:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776585.019313:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776585.019317:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776585.019322:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13467 00000400:00000200:3.0:1775776585.019330:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776585.019335:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776585.019337:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776585.019339:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776585.019342:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776585.019344:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776585.019345:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776585.019348:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776585.019349:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776585.019351:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776585.019353:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776585.019355:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776585.019356:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776585.019359:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776585.019360:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776585.019362:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776585.019364:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776585.019366:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776585.019368:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776585.019369:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776585.019371:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776585.019373:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776585.019375:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776585.019377:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776585.019378:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776585.019383:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775776585.019385:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776585.019388:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776585.019390:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776585.019392:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776585.019394:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776585.019397:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776585.019402:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000001:3.0:1775776585.019407:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776585.019413:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710425344/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776585.019420:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776585.019424:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776585.019427:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710425344/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776585.019440:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776585.019442:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776585.019444:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776585.019445:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776585.019446:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776585.019448:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776585.019450:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13467 00000400:00000200:3.0:1775776585.019453:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776585.019456:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776585.019459:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776585.019462:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776585.019464:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776585.019466:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776585.019468:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:1.0:1775776585.019470:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1775776585.019470:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776585.019472:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:1.0:1775776585.019475:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1775776585.019475:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776585.019477:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:1.0:1775776585.019478:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1775776585.019481:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776585.019486:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:1.0:1775776585.019487:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71500 x1862022710425344/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776585.019496:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776585.019502:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776585.019504:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776585.019505:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776585.019510:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776585.019513:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776585.019514:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:1.0:1775776585.019516:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1775776585.019517:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000001:1.0:1775776585.019520:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776585.019521:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776585.019524:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776585.019524:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1775776585.019530:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710425344:192.168.201.115@tcp:250:kworker.0 02000000:00000001:3.0:1775776585.019532:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776585.019537:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776585.019537:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776585.019541:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776585.019545:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775776585.019546:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776585.019550:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775776585.019550:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776585.019555:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776585.019556:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:3.0:1775776585.019561:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 02000000:00000010:1.0:1775776585.019563:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 00000020:00000040:3.0:1775776585.019564:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776585.019568:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776585.019568:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776585.019570:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776585.019572:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776585.019573:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776585.019575:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776585.019578:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000010:3.0:1775776585.019579:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000100:00000001:3.0:1775776585.019582:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776585.019583:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00100000:3.0:1775776585.019587:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710425472/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1775776585.019592:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000001:3.0:1775776585.019593:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776585.019596:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776585.019599:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710425472/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776585.019601:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710425344, portal 25 00000100:00000001:1.0:1775776585.019606:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776585.019609:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849009408 : -123823860542208 : ffff8f6202c71500) 00000100:00000001:3.0:1775776585.019616:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776585.019619:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71500 x1862022710425344/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775776585.019621:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776585.019631:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776585.019634:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776585.019636:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1775776585.019637:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1775776585.019642:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776585.019642:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000400:00000010:1.0:1775776585.019647:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00080000:3.0:1775776585.019647:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13467 00000100:00000200:1.0:1775776585.019652:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710425344, offset 0 00000400:00000200:3.0:1775776585.019654:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776585.019660:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776585.019663:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1775776585.019664:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776585.019669:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776585.019672:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776585.019675:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775776585.019675:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776585.019679:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776585.019682:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776585.019683:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000400:00000200:1.0:1775776585.019685:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000020:00000001:3.0:1775776585.019686:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776585.019690:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000400:00000200:1.0:1775776585.019692:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1775776585.019693:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776585.019698:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000020:00000001:3.0:1775776585.019698:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776585.019703:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000040:3.0:1775776585.019704:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776585.019708:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:3.0:1775776585.019708:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776585.019712:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776585.019714:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776585.019718:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453875:453875:256:4294967295] 192.168.201.115@tcp LPNI seq info [453800:453800:8:4294967295] 00000020:00000040:3.0:1775776585.019718:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776585.019723:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776585.019726:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776585.019729:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776585.019730:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776585.019732:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776585.019734:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776585.019736:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776585.019737:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776585.019741:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776585.019744:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776585.019747:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000010:3.0:1775776585.019748:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000020:00000040:3.0:1775776585.019752:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776585.019755:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776585.019757:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776585.019758:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776585.019759:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776585.019761:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776585.019763:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775776585.019766:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 02000000:00000010:3.0:1775776585.019768:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000001:3.0:1775776585.019773:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776585.019774:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00100000:3.0:1775776585.019779:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710425600/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1775776585.019780:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775776585.019791:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775776585.019793:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.1:1775776585.019796:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000100:00000001:3.0:1775776585.019797:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776585.019804:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710425600/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1775776585.019805:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776585.019809:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776585.019816:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1775776585.019816:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776585.019822:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776585.019825:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776585.019832:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776585.019834:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000040:3.0:1775776585.019838:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776585.019839:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1775776585.019844:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1775776585.019846:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71500 x1862022710425344/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776585.019850:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776585.019858:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13477) 00000100:00000001:1.0:1775776585.019860:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776585.019865:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71500 x1862022710425344/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776610 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.019878:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776585.019880:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776585.019885:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776585.019889:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776585.019891:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308143878 00000100:00000010:1.0:1775776585.019896:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e29c0. 00000100:00000001:1.0:1775776585.019903:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776585.019908:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308143878 00000100:00000010:1.0:1775776585.019911:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e21e0. 00000100:00000001:1.0:1775776585.019914:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776585.019917:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776585.019920:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776585.019922:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776585.019926:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776585/real 1775776585] req@ffff8f6202c71500 x1862022710425344/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776585.019935:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776585.019937:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776585.019939:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776585.019942:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776585.019943:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776585.019946:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776585.019964:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71500 x1862022710425344/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776585.019973:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71500 x1862022710425344/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.019980:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776585.019982:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776585.019984:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776585.019986:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776585.019990:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71500 x1862022710425344/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.019998:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776585.020003:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71500 x1862022710425344/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.020010:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776585.020013:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776585.020015:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776585.020016:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776585.020018:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776585.020020:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776585.020023:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776585.020027:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776585.020029:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776585.020033:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71500 x1862022710425344/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776585.020041:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710425344:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776585.020046:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776585.020047:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776585.020050:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71500 x1862022710425344/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.020056:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776585.020058:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776585.020060:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776585.020063:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776585.020065:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776585.020066:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776585.020068:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776585.020070:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37c00. 02000000:00000001:1.0:1775776585.020073:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776585.020075:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776585.020077:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71500. 00000100:00000001:1.0:1775776585.020080:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776585.020081:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776585.020083:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776585.020090:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776585.020092:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776585.020093:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776585.020095:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776585.020096:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776585.020099:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70a80 x1862022710425472/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.020106:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776585.020107:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776585.020109:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776585.020111:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776585.020113:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776585.020115:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710425472:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776585.020118:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776585.020120:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776585.020121:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776585.020123:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776585.020126:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776585.020127:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776585.020131:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776585.020162:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776585.020167:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776585.020169:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710425472, portal 10 00000100:00000001:1.0:1775776585.020171:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776585.020173:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849006720 : -123823860544896 : ffff8f6202c70a80) 00000100:00000040:1.0:1775776585.020179:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70a80 x1862022710425472/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.020186:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776585.020188:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776585.020191:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776585.020193:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710425472, offset 0 00000400:00000200:1.0:1775776585.020197:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776585.020201:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776585.020205:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776585.020207:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776585.020210:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776585.020213:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776585.020215:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776585.020218:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453876:453876:256:4294967295] 192.168.201.115@tcp LPNI seq info [453801:453801:8:4294967295] 00000400:00000200:1.0:1775776585.020222:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776585.020227:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776585.020232:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776585.020235:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776585.020239:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776585.020241:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775776585.020245:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776585.020247:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775776585.020250:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776585.020252:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776585.020255:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776585.020259:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776585.020261:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776585.020263:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776585.020265:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776585.020268:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70a80 x1862022710425472/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.020276:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776585.020278:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70a80 x1862022710425472/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776610 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.020284:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776585.020286:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776585.020289:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776585.020293:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776585.020296:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776585.020298:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308140044 00000100:00000001:1.0:1775776585.020300:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308143878 00000100:00000001:1.0:1775776585.020302:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308140044 00000100:00000001:1.0:1775776585.020304:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308143878 00000100:00000001:1.0:1775776585.020306:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776585.020308:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776585.020310:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776585.020313:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776585/real 1775776585] req@ffff8f6202c70a80 x1862022710425472/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776585.020320:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776585.020321:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776585.020323:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776585.020325:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776585.020326:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776585.020328:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776585.020331:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70a80 x1862022710425472/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776585.020338:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70a80 x1862022710425472/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.020344:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776585.020345:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776585.020347:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776585.020348:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776585.020351:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70a80 x1862022710425472/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.020357:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776585.020360:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70a80 x1862022710425472/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.020369:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776585.020372:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776585.020373:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776585.020375:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776585.020376:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776585.020382:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776585.020384:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776585.020387:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776585.020390:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776585.020394:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70a80 x1862022710425472/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776585.020401:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710425472:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776585.020405:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776585.020408:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70700 x1862022710425600/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.020414:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776585.020415:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776585.020417:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776585.020419:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776585.020421:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776585.020424:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710425600:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776585.020427:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776585.020428:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776585.020430:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776585.020431:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776585.020434:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776585.020436:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776585.020439:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776585.020442:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776585.020445:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776585.020447:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710425600, portal 10 00000100:00000001:1.0:1775776585.020449:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776585.020451:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849005824 : -123823860545792 : ffff8f6202c70700) 00000100:00000040:1.0:1775776585.020455:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70700 x1862022710425600/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.020461:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776585.020464:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776585.020466:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776585.020468:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710425600, offset 0 00000400:00000200:1.0:1775776585.020472:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776585.020475:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776585.020478:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776585.020481:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776585.020483:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776585.020486:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776585.020488:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776585.020491:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453877:453877:256:4294967295] 192.168.201.115@tcp LPNI seq info [453802:453802:8:4294967295] 00000400:00000200:1.0:1775776585.020495:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776585.020501:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776585.020505:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776585.020508:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776585.020512:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776585.020514:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1775776585.020518:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776585.020520:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1775776585.020524:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776585.020525:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776585.020528:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776585.020531:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776585.020534:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776585.020536:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776585.020539:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776585.020541:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70700 x1862022710425600/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.020548:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776585.020551:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70700 x1862022710425600/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776610 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.020558:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776585.020559:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776585.020562:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776585.020564:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776585.020567:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776585.020569:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308140044 00000100:00000001:1.0:1775776585.020572:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308143878 00000100:00000001:1.0:1775776585.020574:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308140044 00000100:00000001:1.0:1775776585.020576:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308143878 00000100:00000001:1.0:1775776585.020577:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776585.020579:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776585.020581:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776585.020584:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776585/real 1775776585] req@ffff8f6202c70700 x1862022710425600/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776585.020592:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776585.020593:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776585.020594:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776585.020597:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776585.020598:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776585.020600:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776585.020603:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70700 x1862022710425600/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776585.020611:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70700 x1862022710425600/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.020618:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776585.020619:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776585.020620:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776585.020622:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776585.020624:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70700 x1862022710425600/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.020630:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776585.020634:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70700 x1862022710425600/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.020640:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776585.020642:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776585.020643:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776585.020644:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776585.020646:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776585.020648:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776585.020650:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776585.020653:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776585.020655:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776585.020658:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70700 x1862022710425600/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776585.020665:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710425600:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776585.020669:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776585.020670:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776585.020673:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70a80 x1862022710425472/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.020679:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776585.020681:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776585.020683:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776585.020685:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776585.020687:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776585.020688:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776585.020691:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776585.020692:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35000. 02000000:00000001:1.0:1775776585.020694:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776585.020695:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776585.020697:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70a80. 00000100:00000001:1.0:1775776585.020699:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776585.020700:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776585.020701:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776585.020704:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70700 x1862022710425600/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776610 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776585.020710:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776585.020711:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776585.020712:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776585.020714:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776585.020716:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776585.020717:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776585.020718:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776585.020720:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37800. 02000000:00000001:1.0:1775776585.020721:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776585.020722:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776585.020724:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70700. 00000100:00000001:1.0:1775776585.020726:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776585.020727:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776585.020729:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776585.020732:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776585.020733:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776585.020735:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776585.020737:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776585.147260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776585.147267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776585.147272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776585.147273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776585.147277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776585.147278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776585.147338:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776585.147342:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776585.147345:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776586.044456:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776586.044463:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776586.044465:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776586.044470:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776586.044474:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776586.044477:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776586.044480:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776586.044482:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776586.044485:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776586.171314:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776586.171322:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776586.171329:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776586.171330:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776586.171335:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776586.171336:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776586.171848:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776586.171853:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776586.171858:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776587.067275:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776587.067281:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776587.067283:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776587.067289:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776587.067292:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776587.067295:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776587.067298:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776587.067300:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776587.067303:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776587.196120:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776587.196128:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776587.196130:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776587.196159:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776587.196163:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776587.196166:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776587.196167:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776587.196170:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776587.196174:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776587.196176:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776587.196179:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776587.196180:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776587.196217:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776587.196221:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776587.196224:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776588.091287:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776588.091292:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776588.091294:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776588.091299:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776588.091302:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776588.091305:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776588.091308:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776588.091310:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776588.091313:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776588.219361:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776588.219367:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776588.219372:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776588.219373:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776588.219376:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776588.219377:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776588.219568:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776588.219571:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776588.219574:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776588.923263:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308143878, at 4308143936 00000100:00000010:0.0:1775776588.923277:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e29c0. 00000100:00000040:1.0:1775776588.925013:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308143878, at 4308143937 00000100:00000010:1.0:1775776588.925028:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e21e0. 00000100:00000001:2.0:1775776589.115265:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776589.115272:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776589.115274:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776589.115280:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776589.115283:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776589.115288:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776589.115291:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776589.115294:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776589.115296:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776589.243292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776589.243300:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776589.243315:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776589.243316:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776589.243320:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776589.243321:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776589.243416:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776589.243422:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776589.243427:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775776590.139269:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776590.139279:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776590.139281:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776590.139286:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776590.139291:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775776590.139294:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1775776590.139302:0:141703:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13472 00000100:00000001:3.0:1775776590.139303:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776590.139306:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776590.139311:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000400:00000200:2.0:1775776590.139319:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776590.139320:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776590.139326:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776590.139329:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:2.0:1775776590.139331:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:3.0:1775776590.139332:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776590.139340:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776590.139341:0:141703:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776590.139352:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776590.139356:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776590.139369:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776590.139372:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776590.139376:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776590.139378:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776590.139381:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776590.139384:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776590.139386:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776590.139388:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776590.139392:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776590.139394:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776590.139396:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776590.139397:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776590.139400:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776590.139403:0:141703:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776590.139406:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776590.139409:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776590.139412:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776590.139415:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776590.139418:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776590.139420:0:141703:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776590.139430:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775776590.139434:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776590.139438:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776590.139439:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776590.139442:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776590.139444:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776590.139448:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776590.139455:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775776590.139461:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776590.139469:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710425728/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776590.139479:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776590.139483:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776590.139488:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710425728/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.139553:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776590.139558:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776590.139560:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776590.139565:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344aa00 x1862022710425728/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.139574:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776590.139576:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776590.139579:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776590.139581:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776590.139584:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776590.139587:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710425728:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776590.139591:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776590.139595:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776590.139597:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776590.139599:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776590.139604:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776590.139606:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776590.139611:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776590.139615:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776590.139621:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776590.139628:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710425728, portal 25 00000100:00000001:1.0:1775776590.139630:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776590.139632:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662545920 : -123823047005696 : ffff8f623344aa00) 00000100:00000040:1.0:1775776590.139636:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344aa00 x1862022710425728/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776615 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.139644:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776590.139647:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776590.139649:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776590.139652:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710425728, offset 0 00000400:00000200:1.0:1775776590.139659:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776590.139667:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776590.139673:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776590.139678:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776590.139682:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776590.139686:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776590.139688:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776590.139693:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453878:453878:256:4294967295] 192.168.201.115@tcp LPNI seq info [453803:453803:8:4294967295] 00000400:00000200:1.0:1775776590.139697:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776590.139708:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776590.139714:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776590.139719:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776590.139726:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776590.139729:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775776590.139735:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776590.139738:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775776590.139742:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776590.139744:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776590.139750:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776590.139755:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776590.139760:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776590.139763:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776590.139769:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776590.139772:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344aa00 x1862022710425728/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776615 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776590.139797:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776590.139800:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776590.139804:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776590.139806:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776590.139807:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776590.139810:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776590.139812:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13472 00000400:00000200:2.0:1775776590.139815:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776590.139818:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776590.139821:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776590.139823:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776590.139825:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776590.139827:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776590.139829:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776590.139832:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776590.139833:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776590.139835:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776590.139837:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776590.139839:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776590.139841:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776590.139844:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776590.139846:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776590.139848:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776590.139850:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776590.139852:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776590.139854:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776590.139856:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776590.139857:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776590.139885:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776590.139887:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776590.139889:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776590.139893:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776590.139896:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776590.139898:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776590.139902:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775776590.139904:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776590.139907:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776590.139908:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776590.139910:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776590.139911:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776590.139913:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776590.139917:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775776590.139920:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776590.139925:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710425856/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776590.139933:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776590.139935:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776590.139939:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710425856/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.140021:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776590.140026:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344aa00 x1862022710425728/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776615 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.140041:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776590.140042:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776590.140048:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776590.140052:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776590.140055:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308148998 00000100:00000010:1.0:1775776590.140060:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e21e0. 00000100:00000001:1.0:1775776590.140066:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776590.140087:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308148998 00000100:00000010:1.0:1775776590.140090:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2cc0. 00000100:00000001:1.0:1775776590.140093:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776590.140097:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776590.140099:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776590.140102:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776590.140106:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776590/real 1775776590] req@ffff8f623344aa00 x1862022710425728/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776590.140119:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776590.140121:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776590.140123:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776590.140126:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776590.140128:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776590.140130:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776590.140160:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344aa00 x1862022710425728/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776590.140169:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344aa00 x1862022710425728/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.140176:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776590.140177:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776590.140179:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776590.140181:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776590.140184:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344aa00 x1862022710425728/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.140191:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776590.140195:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344aa00 x1862022710425728/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.140202:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776590.140205:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776590.140206:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776590.140208:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776590.140210:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776590.140212:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776590.140214:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776590.140217:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776590.140219:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776590.140224:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344aa00 x1862022710425728/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776590.140232:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710425728:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776590.140236:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776590.140238:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776590.140240:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344aa00 x1862022710425728/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.140246:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776590.140248:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776590.140250:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776590.140253:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776590.140255:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776590.140256:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776590.140259:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776590.140261:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4c000. 02000000:00000001:1.0:1775776590.140264:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776590.140266:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776590.140268:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344aa00. 00000100:00000001:1.0:1775776590.140271:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776590.140272:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776590.140274:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776590.140281:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776590.140282:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776590.140284:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776590.140286:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776590.140287:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776590.140290:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449500 x1862022710425856/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.140297:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776590.140298:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776590.140299:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776590.140301:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776590.140303:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776590.140306:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710425856:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776590.140309:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776590.140310:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776590.140312:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776590.140313:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776590.140316:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776590.140318:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776590.140321:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776590.140324:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776590.140327:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776590.140329:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710425856, portal 10 00000100:00000001:1.0:1775776590.140331:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776590.140332:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662540544 : -123823047011072 : ffff8f6233449500) 00000100:00000040:1.0:1775776590.140338:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449500 x1862022710425856/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776615 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.140344:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776590.140347:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776590.140349:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776590.140351:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710425856, offset 0 00000400:00000200:1.0:1775776590.140354:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776590.140418:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776590.140421:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776590.140424:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776590.140426:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776590.140429:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776590.140431:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776590.140434:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453879:453879:256:4294967295] 192.168.201.115@tcp LPNI seq info [453804:453804:8:4294967295] 00000400:00000200:1.0:1775776590.140438:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776590.140444:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776590.140449:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776590.140452:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776590.140456:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776590.140458:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775776590.140463:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776590.140465:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775776590.140469:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776590.140470:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776590.140473:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776590.140477:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776590.140479:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776590.140481:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776590.140483:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776590.140487:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449500 x1862022710425856/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776615 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.140495:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776590.140497:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449500 x1862022710425856/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776615 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.140503:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776590.140505:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776590.140508:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776590.140511:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776590.140514:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776590.140516:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308145163 00000100:00000001:1.0:1775776590.140519:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308148998 00000100:00000001:1.0:1775776590.140521:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308145163 00000100:00000001:1.0:1775776590.140523:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308148998 00000100:00000001:1.0:1775776590.140525:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776590.140527:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776590.140529:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776590.140532:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776590/real 1775776590] req@ffff8f6233449500 x1862022710425856/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776590.140539:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776590.140541:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776590.140542:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776590.140544:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776590.140546:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776590.140548:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776590.140551:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449500 x1862022710425856/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776590.140558:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449500 x1862022710425856/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.140564:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776590.140566:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776590.140567:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776590.140568:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776590.140571:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449500 x1862022710425856/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.140578:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776590.140581:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449500 x1862022710425856/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.140587:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776590.140589:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776590.140591:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776590.140592:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776590.140594:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776590.140598:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776590.140601:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776590.140604:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776590.140606:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776590.140610:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449500 x1862022710425856/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776590.140617:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710425856:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776590.140621:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776590.140622:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776590.140625:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449500 x1862022710425856/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.140631:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776590.140632:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776590.140634:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776590.140636:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776590.140637:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776590.140638:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776590.140640:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776590.140642:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4e000. 02000000:00000001:1.0:1775776590.140644:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776590.140645:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776590.140646:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449500. 00000100:00000001:1.0:1775776590.140648:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776590.140649:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776590.140651:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776590.140654:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776590.140655:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776590.140657:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776590.140658:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776590.140814:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776590.140818:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776590.140821:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776590.140823:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776590.140825:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776590.140827:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776590.140830:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13472 00000400:00000200:2.0:1775776590.140834:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776590.140837:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776590.140839:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776590.140842:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776590.140844:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776590.140846:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776590.140848:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776590.140850:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776590.140852:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776590.140853:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776590.140855:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776590.140857:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776590.140870:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776590.140873:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776590.140875:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776590.140877:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776590.140879:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776590.140881:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776590.140884:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776590.140887:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776590.140889:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776590.140892:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776590.140894:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776590.140896:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776590.140898:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776590.140900:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776590.140902:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776590.140906:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775776590.140908:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776590.140910:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776590.140912:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776590.140913:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776590.140914:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776590.140916:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776590.140920:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775776590.140923:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776590.140932:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710425984/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776590.140940:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776590.140944:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776590.140948:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710425984/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776590.140956:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776590.140958:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776590.140963:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776590.140966:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776590.140969:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776590.140973:0:141703:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13482) 00000100:00000001:1.0:1775776590.141021:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776590.141023:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776590.141024:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776590.141028:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344ad80 x1862022710425984/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.141035:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776590.141037:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776590.141039:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776590.141041:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776590.141042:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776590.141045:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710425984:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776590.141048:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776590.141050:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776590.141051:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776590.141053:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776590.141056:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776590.141057:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776590.141060:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776590.141063:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776590.141066:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776590.141068:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710425984, portal 10 00000100:00000001:1.0:1775776590.141070:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776590.141071:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662546816 : -123823047004800 : ffff8f623344ad80) 00000100:00000040:1.0:1775776590.141077:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344ad80 x1862022710425984/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776615 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.141082:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776590.141085:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776590.141086:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776590.141089:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710425984, offset 0 00000400:00000200:1.0:1775776590.141092:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776590.141096:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776590.141099:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776590.141102:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776590.141104:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776590.141106:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776590.141109:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776590.141112:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453880:453880:256:4294967295] 192.168.201.115@tcp LPNI seq info [453805:453805:8:4294967295] 00000400:00000200:1.0:1775776590.141116:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776590.141123:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776590.141127:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776590.141130:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776590.141148:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776590.141151:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1775776590.141155:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776590.141157:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1775776590.141161:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776590.141163:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776590.141166:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776590.141169:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776590.141171:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776590.141177:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776590.141180:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776590.141184:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344ad80 x1862022710425984/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776615 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.141191:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776590.141194:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344ad80 x1862022710425984/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776615 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.141200:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776590.141202:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776590.141204:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776590.141207:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776590.141210:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776590.141212:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308145165 00000100:00000001:1.0:1775776590.141214:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308148998 00000100:00000001:1.0:1775776590.141220:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308145165 00000100:00000001:1.0:1775776590.141222:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308148998 00000100:00000001:1.0:1775776590.141225:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776590.141226:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776590.141228:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776590.141231:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776590/real 1775776590] req@ffff8f623344ad80 x1862022710425984/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776590.141239:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776590.141240:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776590.141244:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776590.141246:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776590.141247:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776590.141249:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776590.141252:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344ad80 x1862022710425984/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776590.141261:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344ad80 x1862022710425984/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.141267:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776590.141268:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776590.141270:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776590.141271:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776590.141274:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344ad80 x1862022710425984/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.141281:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776590.141284:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344ad80 x1862022710425984/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.141291:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776590.141294:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776590.141295:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776590.141296:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776590.141298:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776590.141299:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776590.141302:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776590.141305:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776590.141307:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776590.141311:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344ad80 x1862022710425984/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776590.141332:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710425984:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776590.141336:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776590.141337:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776590.141340:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344ad80 x1862022710425984/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776615 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776590.141346:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776590.141348:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776590.141350:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776590.141352:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776590.141353:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776590.141355:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776590.141356:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776590.141358:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4f000. 02000000:00000001:1.0:1775776590.141420:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776590.141421:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776590.141422:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344ad80. 00000100:00000001:1.0:1775776590.141425:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776590.141426:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776590.141428:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776590.141431:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776590.141432:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776590.141434:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776590.141436:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776590.267220:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776590.267228:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776590.267234:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776590.267235:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776590.267240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776590.267241:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776590.267310:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776590.267315:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776590.267318:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776591.163260:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776591.163268:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776591.163269:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776591.163275:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776591.163278:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776591.163282:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776591.163284:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776591.163287:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776591.163289:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776591.291280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776591.291286:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776591.291291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776591.291294:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776591.291297:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776591.291300:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776591.291303:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776591.291306:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776591.291307:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776591.291310:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776591.291314:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776591.291315:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776591.291316:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776591.291318:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776591.291319:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776591.291320:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776591.291322:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776591.291323:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776591.291411:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776591.291417:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776591.291421:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776592.187234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776592.187239:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776592.187240:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776592.187244:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776592.187247:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776592.187250:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776592.187252:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776592.187254:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776592.187255:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776592.315267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776592.315272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776592.315278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776592.315279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776592.315282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776592.315283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776592.315408:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776592.315412:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776592.315415:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776593.211888:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776593.211894:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776593.211896:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776593.211901:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776593.211905:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776593.211908:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776593.211911:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776593.211913:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776593.211915:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776593.339248:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776593.339252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776593.339255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776593.339256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776593.339259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776593.339259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776593.339542:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776593.339545:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776593.339548:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776594.043249:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308148998, at 4308149056 00000100:00000010:0.0:1775776594.043263:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e21e0. 00000100:00000040:1.0:1775776594.043367:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308148998, at 4308149056 00000100:00000010:1.0:1775776594.043382:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2cc0. 00000100:00000001:3.0:1775776594.235747:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776594.235753:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776594.235756:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776594.235760:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776594.235764:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776594.235767:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776594.235770:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776594.235772:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776594.235775:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776594.363292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776594.363298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776594.363303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776594.363304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776594.363309:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776594.363310:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776594.363406:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776594.363410:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776594.363413:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775776595.259248:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776595.259256:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776595.259258:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775776595.259261:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1775776595.259262:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776595.259267:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776595.259268:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775776595.259271:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776595.259276:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:2.0:1775776595.259277:0:141703:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13477 00000100:00000001:3.0:1775776595.259280:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776595.259284:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776595.259287:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:2.0:1775776595.259288:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776595.259291:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776595.259294:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776595.259297:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776595.259300:0:141703:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776595.259303:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776595.259306:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776595.259309:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776595.259310:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776595.259314:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776595.259316:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776595.259319:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776595.259321:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776595.259323:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776595.259325:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776595.259327:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776595.259329:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776595.259330:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776595.259332:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776595.259334:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776595.259337:0:141703:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776595.259339:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776595.259342:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776595.259345:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776595.259348:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776595.259350:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776595.259352:0:141703:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776595.259360:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775776595.259364:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776595.259377:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776595.259379:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776595.259381:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776595.259383:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776595.259387:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776595.259394:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775776595.259401:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776595.259408:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710426112/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776595.259418:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776595.259422:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776595.259426:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710426112/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776595.259446:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776595.259449:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776595.259452:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776595.259453:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776595.259455:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776595.259457:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776595.259460:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13477 00000400:00000200:2.0:1775776595.259463:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776595.259475:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776595.259477:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776595.259480:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776595.259482:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776595.259484:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776595.259487:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776595.259489:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775776595.259490:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:2.0:1775776595.259491:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:1.0:1775776595.259495:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1775776595.259495:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776595.259497:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:1.0:1775776595.259498:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1775776595.259499:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776595.259503:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:1.0:1775776595.259508:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449180 x1862022710426112/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775776595.259508:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776595.259511:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776595.259513:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776595.259515:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776595.259517:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776595.259520:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776595.259524:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776595.259526:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776595.259531:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776595.259534:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1775776595.259535:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776595.259537:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776595.259538:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776595.259543:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:2.0:1775776595.259544:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775776595.259546:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1775776595.259549:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776595.259551:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776595.259552:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776595.259557:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000100:00100000:1.0:1775776595.259558:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710426112:192.168.201.115@tcp:250:kworker.0 00000020:00000040:2.0:1775776595.259561:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00000001:1.0:1775776595.259566:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776595.259566:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776595.259569:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776595.259571:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776595.259572:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776595.259574:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776595.259579:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000010:2.0:1775776595.259579:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 02000000:00000001:1.0:1775776595.259584:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776595.259584:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776595.259588:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:2.0:1775776595.259592:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710426240/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1775776595.259596:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776595.259600:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776595.259610:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:2.0:1775776595.259611:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776595.259614:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776595.259617:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000040:2.0:1775776595.259618:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710426240/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1775776595.259625:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776595.259633:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710426112, portal 25 00000100:00000001:1.0:1775776595.259637:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776595.259637:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776595.259640:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662539648 : -123823047011968 : ffff8f6233449180) 00000100:00080000:2.0:1775776595.259642:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776595.259646:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776595.259648:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:1.0:1775776595.259650:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449180 x1862022710426112/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775776595.259651:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776595.259657:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776595.259662:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13477 00000100:00000001:1.0:1775776595.259666:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:2.0:1775776595.259669:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:1.0:1775776595.259673:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000200:2.0:1775776595.259675:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776595.259677:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776595.259680:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000010:1.0:1775776595.259683:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000001:2.0:1775776595.259684:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776595.259688:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000200:1.0:1775776595.259689:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710426112, offset 0 00000100:00000001:2.0:1775776595.259690:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776595.259694:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776595.259695:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776595.259696:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776595.259698:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776595.259701:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776595.259702:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000020:00000001:2.0:1775776595.259705:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776595.259709:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776595.259710:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776595.259713:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:2.0:1775776595.259714:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776595.259717:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776595.259720:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776595.259723:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00080000:2.0:1775776595.259725:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776595.259730:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776595.259731:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:2.0:1775776595.259732:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776595.259736:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775776595.259738:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 02000000:00000001:2.0:1775776595.259740:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776595.259743:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776595.259745:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1775776595.259746:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000008:00000001:2.0:1775776595.259749:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776595.259752:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:2.0:1775776595.259753:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776595.259758:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000400:00000200:1.0:1775776595.259760:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453881:453881:256:4294967295] 192.168.201.115@tcp LPNI seq info [453806:453806:8:4294967295] 00000020:00000040:2.0:1775776595.259762:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776595.259768:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1775776595.259781:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776595.259791:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 02000000:00000001:2.0:1775776595.259791:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776595.259795:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776595.259798:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776595.259800:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776595.259803:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 02000000:00000010:2.0:1775776595.259805:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d800. 00000800:00000010:1.0:1775776595.259810:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000100:00000001:2.0:1775776595.259810:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776595.259817:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710426368/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1775776595.259821:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776595.259828:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1775776595.259831:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776595.259834:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776595.259837:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710426368/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776595.259846:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776595.259847:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:2.0:1775776595.259850:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1775776595.259853:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000100:00000040:2.0:1775776595.259855:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776595.259860:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1775776595.259864:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000040:2.0:1775776595.259866:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000010:1.0:1775776595.259868:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000100:00000040:2.0:1775776595.259873:0:141703:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13487) 00000400:00000200:1.0:1775776595.259875:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776595.259882:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776595.259888:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776595.259890:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776595.259896:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776595.259901:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449180 x1862022710426112/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.259909:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776595.259912:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449180 x1862022710426112/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776620 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.259919:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776595.259921:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776595.259926:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776595.259930:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776595.259939:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308154118 00000100:00000010:1.0:1775776595.259945:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2cc0. 00000100:00000001:1.0:1775776595.259951:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776595.259956:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308154118 00000100:00000010:1.0:1775776595.259959:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2d20. 00000100:00000001:1.0:1775776595.259963:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776595.259965:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776595.259968:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776595.259970:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776595.259974:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776595/real 1775776595] req@ffff8f6233449180 x1862022710426112/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776595.259983:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776595.259985:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776595.259987:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776595.259989:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776595.259991:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776595.259993:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776595.259997:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449180 x1862022710426112/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776595.260011:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449180 x1862022710426112/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.260017:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776595.260019:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776595.260021:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776595.260022:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776595.260026:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449180 x1862022710426112/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.260032:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776595.260036:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449180 x1862022710426112/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.260042:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776595.260046:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776595.260047:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776595.260048:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776595.260050:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776595.260052:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776595.260054:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776595.260057:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776595.260059:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776595.260062:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449180 x1862022710426112/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776595.260070:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710426112:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776595.260075:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776595.260076:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776595.260079:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449180 x1862022710426112/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.260084:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776595.260086:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776595.260088:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776595.260100:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776595.260101:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776595.260103:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776595.260105:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776595.260106:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4cc00. 02000000:00000001:1.0:1775776595.260110:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776595.260111:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776595.260113:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449180. 00000100:00000001:1.0:1775776595.260115:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776595.260117:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776595.260118:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776595.260124:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776595.260126:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776595.260128:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776595.260129:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776595.260131:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776595.260160:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449c00 x1862022710426240/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.260166:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776595.260168:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776595.260170:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776595.260171:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776595.260173:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776595.260175:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710426240:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776595.260178:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776595.260180:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776595.260181:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776595.260182:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776595.260185:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776595.260187:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776595.260190:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776595.260192:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776595.260195:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776595.260198:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710426240, portal 10 00000100:00000001:1.0:1775776595.260200:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776595.260201:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662542336 : -123823047009280 : ffff8f6233449c00) 00000100:00000040:1.0:1775776595.260206:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449c00 x1862022710426240/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.260212:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776595.260214:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776595.260216:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776595.260218:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710426240, offset 0 00000400:00000200:1.0:1775776595.260221:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776595.260225:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776595.260227:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776595.260230:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776595.260232:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776595.260234:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776595.260236:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776595.260239:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453882:453882:256:4294967295] 192.168.201.115@tcp LPNI seq info [453807:453807:8:4294967295] 00000400:00000200:1.0:1775776595.260243:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776595.260249:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776595.260252:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776595.260255:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776595.260258:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776595.260260:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775776595.260264:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776595.260266:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775776595.260270:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776595.260271:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776595.260274:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776595.260277:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776595.260279:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776595.260281:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776595.260283:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776595.260285:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449c00 x1862022710426240/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.260291:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776595.260294:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449c00 x1862022710426240/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776620 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.260299:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776595.260301:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776595.260303:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776595.260306:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776595.260309:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776595.260311:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308150284 00000100:00000001:1.0:1775776595.260313:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308154118 00000100:00000001:1.0:1775776595.260315:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308150284 00000100:00000001:1.0:1775776595.260317:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308154118 00000100:00000001:1.0:1775776595.260319:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776595.260321:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776595.260322:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776595.260325:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776595/real 1775776595] req@ffff8f6233449c00 x1862022710426240/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776595.260333:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776595.260334:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776595.260335:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776595.260337:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776595.260338:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776595.260340:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776595.260342:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449c00 x1862022710426240/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776595.260350:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449c00 x1862022710426240/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.260363:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776595.260364:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776595.260365:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776595.260366:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776595.260370:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449c00 x1862022710426240/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.260376:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776595.260379:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449c00 x1862022710426240/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.260385:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776595.260388:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776595.260389:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776595.260390:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776595.260392:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776595.260397:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776595.260399:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776595.260402:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776595.260404:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776595.260408:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449c00 x1862022710426240/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776595.260414:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710426240:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776595.260417:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776595.260419:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233448700 x1862022710426368/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.260424:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776595.260426:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776595.260427:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776595.260429:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776595.260431:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776595.260433:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710426368:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776595.260436:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776595.260437:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776595.260438:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776595.260440:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776595.260442:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776595.260444:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776595.260447:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776595.260449:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776595.260451:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776595.260454:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710426368, portal 10 00000100:00000001:1.0:1775776595.260456:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776595.260457:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662536960 : -123823047014656 : ffff8f6233448700) 00000100:00000040:1.0:1775776595.260462:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233448700 x1862022710426368/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.260468:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776595.260470:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776595.260472:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776595.260474:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710426368, offset 0 00000400:00000200:1.0:1775776595.260477:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776595.260480:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776595.260483:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776595.260485:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776595.260487:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776595.260489:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776595.260491:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776595.260494:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453883:453883:256:4294967295] 192.168.201.115@tcp LPNI seq info [453808:453808:8:4294967295] 00000400:00000200:1.0:1775776595.260498:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776595.260503:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776595.260507:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776595.260509:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776595.260512:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776595.260514:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1775776595.260518:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776595.260521:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1775776595.260524:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776595.260526:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776595.260528:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776595.260531:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776595.260533:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776595.260535:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776595.260537:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776595.260539:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233448700 x1862022710426368/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.260547:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776595.260549:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448700 x1862022710426368/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776620 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.260562:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776595.260563:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776595.260565:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776595.260568:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776595.260570:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776595.260572:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308150284 00000100:00000001:1.0:1775776595.260574:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308154118 00000100:00000001:1.0:1775776595.260576:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308150284 00000100:00000001:1.0:1775776595.260577:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308154118 00000100:00000001:1.0:1775776595.260579:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776595.260581:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776595.260582:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776595.260585:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776595/real 1775776595] req@ffff8f6233448700 x1862022710426368/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776595.260591:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776595.260593:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776595.260594:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776595.260596:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776595.260597:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776595.260599:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776595.260601:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233448700 x1862022710426368/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776595.260608:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233448700 x1862022710426368/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.260614:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776595.260615:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776595.260616:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776595.260617:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776595.260620:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233448700 x1862022710426368/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.260627:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776595.260631:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233448700 x1862022710426368/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.260636:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776595.260638:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776595.260640:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776595.260641:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776595.260642:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776595.260644:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776595.260646:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776595.260648:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776595.260650:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776595.260653:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233448700 x1862022710426368/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776595.260659:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710426368:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776595.260662:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776595.260664:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776595.260666:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449c00 x1862022710426240/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.260673:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776595.260674:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776595.260675:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776595.260677:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776595.260679:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776595.260680:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776595.260681:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776595.260683:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4d400. 02000000:00000001:1.0:1775776595.260685:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776595.260686:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776595.260687:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449c00. 00000100:00000001:1.0:1775776595.260690:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776595.260691:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776595.260692:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776595.260695:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233448700 x1862022710426368/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776620 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776595.260701:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776595.260702:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776595.260703:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776595.260704:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776595.260705:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776595.260706:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776595.260708:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776595.260709:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4d800. 02000000:00000001:1.0:1775776595.260710:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776595.260711:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776595.260712:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233448700. 00000100:00000001:1.0:1775776595.260714:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776595.260715:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776595.260717:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776595.260719:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776595.260720:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776595.260722:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776595.260723:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776595.387228:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776595.387233:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776595.387238:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776595.387239:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776595.387243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776595.387244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776595.387296:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776595.387298:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776595.387301:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776596.283281:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776596.283294:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776596.283296:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776596.283303:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776596.283307:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776596.283312:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776596.283315:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776596.283318:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776596.283321:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776596.411294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776596.411304:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776596.411307:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776596.411311:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776596.411317:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776596.411320:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776596.411322:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776596.411325:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776596.411329:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776596.411330:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776596.411333:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776596.411334:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776596.411396:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776596.411410:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776596.411414:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776597.307276:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776597.307283:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776597.307285:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776597.307291:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776597.307294:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776597.307298:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776597.307301:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776597.307303:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776597.307305:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776597.435212:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776597.435218:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776597.435223:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776597.435224:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776597.435226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776597.435227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776597.435355:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776597.435358:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776597.435361:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776598.331319:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776598.331329:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776598.331331:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776598.331339:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776598.331344:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776598.331349:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776598.331352:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776598.331355:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776598.331358:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776598.461355:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776598.461363:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776598.461370:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776598.461371:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776598.461375:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776598.461376:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776598.461450:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776598.461455:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776598.461459:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776599.163256:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308154118, at 4308154176 00000100:00000040:1.0:1775776599.163257:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308154118, at 4308154176 00000100:00000010:1.0:1775776599.163276:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2d20. 00000100:00000010:0.0:1775776599.163277:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2cc0. 00000100:00000001:3.0:1775776599.355199:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776599.355218:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776599.355220:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776599.355227:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776599.355231:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776599.355235:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776599.355239:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776599.355242:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776599.355246:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776599.483306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776599.483314:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776599.483324:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776599.483325:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776599.483329:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776599.483330:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776599.483476:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776599.483480:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776599.483496:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776600.380827:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776600.380829:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776600.380836:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776600.380840:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776600.380841:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776600.380846:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1775776600.380847:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:3.0:1775776600.380851:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776600.380853:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776600.380858:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775776600.380859:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776600.380864:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776600.380867:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776600.380870:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776600.380879:0:141703:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13482 00000400:00000200:3.0:1775776600.381041:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776600.381049:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776600.381054:0:141703:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776600.381057:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776600.381061:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776600.381063:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776600.381065:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776600.381069:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776600.381072:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776600.381075:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776600.381078:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776600.381081:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776600.381083:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776600.381086:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776600.381088:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776600.381090:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776600.381101:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776600.381104:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776600.381107:0:141703:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776600.381110:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776600.381114:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776600.381116:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776600.381118:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776600.381120:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776600.381123:0:141703:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776600.381147:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775776600.381150:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776600.381155:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776600.381156:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776600.381158:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776600.381161:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776600.381166:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776600.381175:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000001:3.0:1775776600.381182:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776600.381190:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710426496/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776600.381198:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776600.381202:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776600.381206:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710426496/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.381257:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776600.381260:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776600.381262:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776600.381267:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72a00 x1862022710426496/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.381280:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776600.381281:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776600.381285:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776600.381287:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776600.381289:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776600.381291:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710426496:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776600.381295:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776600.381298:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776600.381300:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776600.381302:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776600.381307:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776600.381308:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776600.381314:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776600.381317:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776600.381323:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776600.381331:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710426496, portal 25 00000100:00000001:1.0:1775776600.381332:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776600.381334:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849014784 : -123823860536832 : ffff8f6202c72a00) 00000100:00000040:1.0:1775776600.381337:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72a00 x1862022710426496/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776625 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.381343:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776600.381345:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776600.381347:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776600.381349:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710426496, offset 0 00000400:00000200:1.0:1775776600.381353:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776600.381361:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776600.381366:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776600.381370:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776600.381374:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776600.381378:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776600.381380:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776600.381384:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453884:453884:256:4294967295] 192.168.201.115@tcp LPNI seq info [453809:453809:8:4294967295] 00000400:00000200:1.0:1775776600.381387:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776600.381395:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776600.381399:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776600.381404:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776600.381410:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776600.381414:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775776600.381418:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776600.381420:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1775776600.381423:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776600.381425:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776600.381431:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776600.381435:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776600.381439:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776600.381441:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776600.381447:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776600.381450:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72a00 x1862022710426496/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776625 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776600.381477:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776600.381480:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776600.381482:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776600.381483:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776600.381484:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776600.381487:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776600.381488:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13482 00000400:00000200:3.0:1775776600.381491:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776600.381493:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776600.381495:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776600.381497:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776600.381499:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776600.381500:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776600.381501:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776600.381503:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776600.381504:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776600.381505:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776600.381506:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776600.381507:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776600.381508:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776600.381510:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776600.381511:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776600.381512:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776600.381513:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776600.381515:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776600.381516:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776600.381517:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776600.381518:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776600.381523:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776600.381525:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776600.381526:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776600.381529:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776600.381531:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776600.381532:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776600.381534:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000020:00000040:3.0:1775776600.381536:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776600.381537:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776600.381538:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776600.381539:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776600.381540:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776600.381542:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776600.381544:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:3.0:1775776600.381547:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776600.381550:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710426624/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776600.381557:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776600.381560:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776600.381563:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710426624/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.382022:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776600.382028:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72a00 x1862022710426496/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776625 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.382035:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776600.382037:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776600.382045:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776600.382049:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776600.382053:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308159240 00000100:00000010:1.0:1775776600.382058:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2d20. 00000100:00000001:1.0:1775776600.382066:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776600.382074:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308159240 00000100:00000010:1.0:1775776600.382077:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2ae0. 00000100:00000001:1.0:1775776600.382079:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776600.382084:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776600.382087:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776600.382089:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776600.382094:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776600/real 1775776600] req@ffff8f6202c72a00 x1862022710426496/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776600.382116:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776600.382117:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776600.382120:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776600.382122:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776600.382124:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776600.382127:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776600.382131:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72a00 x1862022710426496/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776600.382171:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72a00 x1862022710426496/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.382179:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776600.382180:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776600.382183:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776600.382185:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776600.382190:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72a00 x1862022710426496/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.382197:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776600.382201:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72a00 x1862022710426496/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.382209:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776600.382212:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776600.382214:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776600.382215:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776600.382218:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776600.382220:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776600.382222:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776600.382226:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776600.382228:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776600.382231:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72a00 x1862022710426496/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776600.382237:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710426496:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776600.382240:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776600.382241:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776600.382243:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72a00 x1862022710426496/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.382247:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776600.382249:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776600.382251:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776600.382253:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776600.382254:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776600.382255:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776600.382256:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776600.382258:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35800. 02000000:00000001:1.0:1775776600.382261:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776600.382262:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776600.382264:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72a00. 00000100:00000001:1.0:1775776600.382266:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776600.382267:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776600.382268:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776600.382284:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776600.382285:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776600.382287:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776600.382288:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776600.382289:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776600.382291:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72300 x1862022710426624/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.382295:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776600.382296:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776600.382298:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776600.382299:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776600.382301:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776600.382303:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710426624:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776600.382305:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776600.382306:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776600.382307:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776600.382308:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776600.382311:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776600.382312:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776600.382315:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776600.382317:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776600.382320:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776600.382322:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710426624, portal 10 00000100:00000001:1.0:1775776600.382323:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776600.382324:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849012992 : -123823860538624 : ffff8f6202c72300) 00000100:00000040:1.0:1775776600.382329:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72300 x1862022710426624/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776625 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.382333:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776600.382334:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776600.382336:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776600.382337:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710426624, offset 0 00000400:00000200:1.0:1775776600.382340:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776600.382342:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776600.382345:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776600.382346:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776600.382349:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776600.382350:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776600.382352:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776600.382354:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453885:453885:256:4294967295] 192.168.201.115@tcp LPNI seq info [453810:453810:8:4294967295] 00000400:00000200:1.0:1775776600.382357:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776600.382361:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776600.382365:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776600.382367:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776600.382369:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776600.382371:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775776600.382374:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776600.382375:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1775776600.382378:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776600.382379:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776600.382381:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776600.382383:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776600.382385:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776600.382386:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776600.382388:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776600.382390:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72300 x1862022710426624/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776625 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.382394:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776600.382396:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72300 x1862022710426624/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776625 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.382399:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776600.382400:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776600.382402:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776600.382404:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776600.382406:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776600.382407:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308155406 00000100:00000001:1.0:1775776600.382409:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308159240 00000100:00000001:1.0:1775776600.382411:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308155406 00000100:00000001:1.0:1775776600.382412:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308159240 00000100:00000001:1.0:1775776600.382413:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776600.382414:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776600.382415:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776600.382417:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776600/real 1775776600] req@ffff8f6202c72300 x1862022710426624/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776600.382423:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776600.382423:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776600.382425:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776600.382426:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776600.382427:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776600.382428:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776600.382429:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72300 x1862022710426624/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776600.382433:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72300 x1862022710426624/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.382437:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776600.382437:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776600.382438:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776600.382439:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776600.382441:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72300 x1862022710426624/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.382444:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776600.382446:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72300 x1862022710426624/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.382449:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776600.382451:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776600.382718:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776600.382719:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776600.382721:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776600.382726:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776600.382729:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776600.382733:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776600.382735:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776600.382741:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72300 x1862022710426624/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776600.382749:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710426624:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776600.382753:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776600.382755:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776600.382758:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72300 x1862022710426624/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.382764:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776600.382765:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776600.382767:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776600.382770:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776600.382771:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776600.382772:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776600.382775:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776600.382776:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35400. 02000000:00000001:1.0:1775776600.382779:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776600.382780:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776600.382781:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72300. 00000100:00000001:1.0:1775776600.382783:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776600.382784:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776600.382786:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776600.382789:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776600.382790:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776600.382792:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776600.382793:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776600.382872:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776600.382875:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776600.382911:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776600.382915:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776600.382918:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776600.382920:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776600.382922:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776600.382925:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776600.382929:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13482 00000400:00000200:3.0:1775776600.382933:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776600.382936:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776600.382938:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776600.382941:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776600.382943:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776600.382945:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776600.382947:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776600.382949:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776600.382951:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776600.382952:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776600.382955:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776600.382957:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776600.382959:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776600.382961:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776600.382963:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776600.382965:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776600.382967:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776600.382970:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776600.382972:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776600.382975:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776600.382977:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776600.382981:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776600.382983:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776600.382985:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776600.382988:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776600.382991:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776600.382992:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776600.382996:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000020:00000040:3.0:1775776600.382999:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776600.383001:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776600.383002:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776600.383004:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776600.383005:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776600.383008:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776600.383012:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000100:00000001:3.0:1775776600.383016:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776600.383020:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710426752/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776600.383028:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776600.383031:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776600.383035:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710426752/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.383083:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776600.383086:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776600.383087:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776600.383092:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71180 x1862022710426752/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.383099:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776600.383101:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776600.383103:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776600.383104:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776600.383106:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776600.383109:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710426752:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776600.383112:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776600.383114:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776600.383115:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776600.383117:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776600.383120:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776600.383122:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776600.383126:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776600.383128:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776600.383131:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776600.383147:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710426752, portal 10 00000100:00000001:1.0:1775776600.383150:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776600.383151:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849008512 : -123823860543104 : ffff8f6202c71180) 00000100:00000040:1.0:1775776600.383157:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71180 x1862022710426752/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776625 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.383163:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776600.383165:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776600.383168:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776600.383170:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710426752, offset 0 00000400:00000200:1.0:1775776600.383175:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776600.383179:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776600.383183:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776600.383186:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776600.383188:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776600.383191:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776600.383194:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776600.383196:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453886:453886:256:4294967295] 192.168.201.115@tcp LPNI seq info [453811:453811:8:4294967295] 00000400:00000200:1.0:1775776600.383200:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776600.383207:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776600.383212:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776600.383215:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776600.383219:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776600.383222:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1775776600.383226:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776600.383229:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1775776600.383233:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776600.383235:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776600.383238:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776600.383241:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776600.383244:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776600.383246:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776600.383249:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776600.383252:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71180 x1862022710426752/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776625 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776600.383303:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776600.383307:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776600.383311:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776600.383314:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776600.383318:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776600.383324:0:141703:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13493) 00000100:00000001:1.0:1775776600.383361:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776600.383366:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71180 x1862022710426752/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776625 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.383373:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776600.383374:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776600.383378:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776600.383381:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776600.383385:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776600.383387:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308155407 00000100:00000001:1.0:1775776600.383389:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308159240 00000100:00000001:1.0:1775776600.383392:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308155407 00000100:00000001:1.0:1775776600.383394:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308159240 00000100:00000001:1.0:1775776600.383396:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776600.383398:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776600.383400:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776600.383403:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776600/real 1775776600] req@ffff8f6202c71180 x1862022710426752/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776600.383411:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776600.383413:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776600.383414:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776600.383417:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776600.383418:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776600.383420:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776600.383423:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71180 x1862022710426752/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776600.383430:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71180 x1862022710426752/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.383437:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776600.383438:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776600.383440:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776600.383441:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776600.383444:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71180 x1862022710426752/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.383451:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776600.383454:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71180 x1862022710426752/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.383461:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776600.383463:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776600.383465:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776600.383466:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776600.383468:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776600.383469:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776600.383472:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776600.383475:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776600.383477:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776600.383481:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71180 x1862022710426752/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776600.383489:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710426752:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776600.383492:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776600.383494:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776600.383496:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71180 x1862022710426752/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776625 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776600.383502:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776600.383504:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776600.383506:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776600.383509:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776600.383510:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776600.383512:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776600.383514:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776600.383515:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37400. 02000000:00000001:1.0:1775776600.383518:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776600.383519:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776600.383520:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71180. 00000100:00000001:1.0:1775776600.383523:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776600.383524:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776600.383526:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776600.383530:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776600.383531:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776600.383533:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776600.383534:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776600.507207:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776600.507214:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776600.507221:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776600.507222:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776600.507226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776600.507227:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776600.507292:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776600.507296:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776600.507298:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776601.403287:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776601.403296:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776601.403298:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776601.403307:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776601.403312:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776601.403317:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776601.403320:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776601.403323:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776601.403327:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776601.531434:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776601.531442:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776601.531448:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776601.531452:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776601.531453:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776601.531456:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776601.531459:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776601.531461:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776601.531462:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776601.531464:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776601.531467:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776601.531468:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776601.531468:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776601.531469:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776601.531470:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776601.531471:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776601.531472:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776601.531473:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776601.531542:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776601.531547:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776601.531550:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776602.428037:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776602.428044:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776602.428046:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776602.428051:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776602.428055:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776602.428059:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776602.428062:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776602.428064:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776602.428067:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776602.555201:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776602.555205:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776602.555209:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776602.555210:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776602.555212:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776602.555213:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776602.555272:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776602.555276:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776602.555280:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776603.451253:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776603.451259:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776603.451260:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776603.451265:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776603.451269:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776603.451273:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776603.451275:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776603.451277:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776603.451279:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776603.579263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776603.579268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776603.579271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776603.579279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776603.579284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776603.579284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776603.579350:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776603.579354:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776603.579357:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776604.283258:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308159240, at 4308159296 00000100:00000040:1.0:1775776604.283258:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308159240, at 4308159296 00000100:00000010:1.0:1775776604.283271:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ae0. 00000100:00000010:0.0:1775776604.283274:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2d20. 00000800:00000200:2.0:1775776604.285877:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:2.0:1775776604.285884:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:2.0:1775776604.285888:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1775776604.286037:0:2404:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1775776604.286042:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776604.286053:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775776604.286057:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776604.286059:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776604.475242:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776604.475250:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776604.475252:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776604.475257:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776604.475261:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776604.475265:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776604.475267:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776604.475269:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776604.475271:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776604.603269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776604.603274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776604.603278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776604.603279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776604.603283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776604.603284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776604.603341:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776604.603344:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776604.603347:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776605.499245:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776605.499250:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775776605.499253:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776605.499256:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776605.499260:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776605.499264:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776605.499264:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775776605.499266:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776605.499270:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:3.0:1775776605.499270:0:141703:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13488 00000100:00000001:0.0:1775776605.499272:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776605.499275:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776605.499278:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000400:00000200:3.0:1775776605.499278:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776605.499280:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776605.499282:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776605.499283:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776605.499286:0:141703:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776605.499289:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776605.499292:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776605.499294:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776605.499296:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776605.499299:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776605.499301:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776605.499303:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776605.499305:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776605.499307:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776605.499310:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776605.499313:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776605.499314:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776605.499316:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776605.499317:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776605.499319:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776605.499321:0:141703:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776605.499323:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776605.499325:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776605.499327:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776605.499329:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776605.499330:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776605.499332:0:141703:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776605.499339:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73b80. 00000020:00000040:3.0:1775776605.499341:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776605.499345:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776605.499346:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776605.499348:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776605.499350:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776605.499353:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776605.499359:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000001:3.0:1775776605.499363:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776605.499370:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73b80 x1862022710426880/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776605.499376:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776605.499379:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776605.499383:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006c52008f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73b80 x1862022710426880/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776605.499407:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776605.499410:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776605.499413:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776605.499414:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776605.499416:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776605.499418:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776605.499421:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13488 00000400:00000200:3.0:1775776605.499424:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776605.499427:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776605.499429:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776605.499432:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776605.499434:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776605.499437:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776605.499439:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776605.499441:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776605.499442:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776605.499444:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776605.499446:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776605.499449:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776605.499451:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776605.499453:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776605.499455:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776605.499457:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1775776605.499457:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776605.499459:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776605.499461:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000001:1.0:1775776605.499462:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776605.499464:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776605.499466:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776605.499467:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776605.499469:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1775776605.499474:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73b80 x1862022710426880/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775776605.499474:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776605.499478:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776605.499482:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776605.499486:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775776605.499491:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000008:00000001:3.0:1775776605.499492:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776605.499495:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776605.499495:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776605.499501:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:3.0:1775776605.499501:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 02000000:00000001:1.0:1775776605.499505:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1775776605.499505:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776605.499509:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776605.499511:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776605.499511:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776605.499515:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:1.0:1775776605.499517:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710426880:192.168.201.115@tcp:250:kworker.0 02000000:00000001:3.0:1775776605.499518:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776605.499522:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776605.499524:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000010:3.0:1775776605.499527:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 02000000:00000001:1.0:1775776605.499528:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776605.499532:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776605.499533:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776605.499536:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1775776605.499541:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710427008/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1775776605.499542:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776605.499546:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776605.499555:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776605.499555:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776605.499560:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776605.499561:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000040:3.0:1775776605.499566:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710427008/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1775776605.499570:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776605.499578:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710426880, portal 25 00000100:00000001:3.0:1775776605.499587:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776605.499588:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776605.499600:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849019264 : -123823860532352 : ffff8f6202c73b80) 00000100:00080000:3.0:1775776605.499604:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1775776605.499608:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73b80 x1862022710426880/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776630 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775776605.499610:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776605.499613:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776605.499615:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776605.499619:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776605.499623:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13488 00000100:00000001:1.0:1775776605.499624:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1775776605.499628:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:1.0:1775776605.499631:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000200:3.0:1775776605.499634:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000400:00000010:1.0:1775776605.499635:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00080000:3.0:1775776605.499636:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776605.499640:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000200:1.0:1775776605.499641:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710426880, offset 0 00000100:00000001:3.0:1775776605.499642:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776605.499645:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776605.499647:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776605.499648:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776605.499649:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776605.499651:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:1.0:1775776605.499652:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000020:00000040:3.0:1775776605.499653:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776605.499657:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776605.499658:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776605.499661:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776605.499662:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776605.499664:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775776605.499666:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776605.499669:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776605.499670:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776605.499674:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000400:00000200:1.0:1775776605.499675:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:3.0:1775776605.499677:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776605.499680:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776605.499681:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1775776605.499684:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775776605.499687:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 02000000:00000001:3.0:1775776605.499688:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776605.499691:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:1.0:1775776605.499693:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000008:00100000:3.0:1775776605.499695:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1775776605.499698:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000008:00000001:3.0:1775776605.499701:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776605.499706:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453887:453887:256:4294967295] 192.168.201.115@tcp LPNI seq info [453812:453812:8:4294967295] 00000100:00000001:3.0:1775776605.499706:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776605.499713:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 00000400:00000200:1.0:1775776605.499714:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000040:3.0:1775776605.499718:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776605.499722:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776605.499725:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776605.499729:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:3.0:1775776605.499729:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776605.499732:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776605.499736:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776605.499740:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 02000000:00000010:3.0:1775776605.499743:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000800:00000010:1.0:1775776605.499746:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000100:00000001:3.0:1775776605.499748:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776605.499755:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710427136/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1775776605.499764:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775776605.499765:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.0:1775776605.499769:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000100:00000001:3.0:1775776605.499770:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776605.499778:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710427136/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1775776605.499779:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776605.499784:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000100:1.0:1775776605.499794:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776605.499797:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775776605.499798:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000100:00000040:3.0:1775776605.499804:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776605.499807:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:3.0:1775776605.499808:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776605.499812:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776605.499815:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000040:3.0:1775776605.499819:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1775776605.499822:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776605.499825:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000040:3.0:1775776605.499825:0:141703:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13498) 00000100:00000001:1.0:1775776605.499834:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776605.499838:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73b80 x1862022710426880/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776630 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.499847:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776605.499851:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73b80 x1862022710426880/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776630 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.499859:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776605.499861:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776605.499865:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776605.499868:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776605.499871:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308164358 00000100:00000010:1.0:1775776605.499875:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2ae0. 00000100:00000001:1.0:1775776605.499881:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776605.499886:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308164358 00000100:00000010:1.0:1775776605.499889:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2c60. 00000100:00000001:1.0:1775776605.499892:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776605.499896:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776605.499898:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776605.499900:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776605.499904:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776605/real 1775776605] req@ffff8f6202c73b80 x1862022710426880/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776605.499913:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776605.499914:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776605.499916:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776605.499919:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776605.499920:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776605.499923:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776605.499926:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73b80 x1862022710426880/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776605.499935:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73b80 x1862022710426880/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.499940:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776605.499942:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776605.499944:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776605.499946:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776605.499950:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73b80 x1862022710426880/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.499956:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776605.499961:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73b80 x1862022710426880/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.499968:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776605.499971:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776605.499972:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776605.499973:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776605.499976:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776605.499978:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776605.499981:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776605.499984:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776605.499987:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776605.499991:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73b80 x1862022710426880/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776605.499998:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710426880:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776605.500002:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776605.500004:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776605.500007:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73b80 x1862022710426880/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.500013:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776605.500015:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776605.500017:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776605.500020:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776605.500022:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776605.500023:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776605.500025:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776605.500027:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37000. 02000000:00000001:1.0:1775776605.500030:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776605.500032:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776605.500034:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73b80. 00000100:00000001:1.0:1775776605.500037:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776605.500038:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776605.500040:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776605.500045:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776605.500056:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776605.500060:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776605.500062:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776605.500063:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776605.500066:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71180 x1862022710427008/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.500079:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776605.500080:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776605.500082:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776605.500084:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776605.500086:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776605.500088:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710427008:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776605.500092:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776605.500093:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776605.500095:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776605.500096:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776605.500099:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776605.500100:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776605.500104:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776605.500106:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776605.500109:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776605.500112:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710427008, portal 10 00000100:00000001:1.0:1775776605.500114:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776605.500115:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849008512 : -123823860543104 : ffff8f6202c71180) 00000100:00000040:1.0:1775776605.500121:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71180 x1862022710427008/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776630 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.500127:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776605.500129:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776605.500131:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776605.500159:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710427008, offset 0 00000400:00000200:1.0:1775776605.500164:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776605.500168:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776605.500171:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776605.500174:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776605.500176:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776605.500179:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776605.500181:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776605.500184:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453888:453888:256:4294967295] 192.168.201.115@tcp LPNI seq info [453813:453813:8:4294967295] 00000400:00000200:1.0:1775776605.500188:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776605.500194:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776605.500198:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776605.500201:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776605.500205:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776605.500208:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775776605.500212:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776605.500215:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000100:1.0:1775776605.500218:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776605.500220:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776605.500222:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776605.500226:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776605.500228:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776605.500230:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776605.500232:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776605.500236:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71180 x1862022710427008/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776630 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.500244:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776605.500246:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71180 x1862022710427008/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776630 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.500253:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776605.500254:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776605.500256:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776605.500260:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776605.500263:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776605.500265:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308160524 00000100:00000001:1.0:1775776605.500268:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308164358 00000100:00000001:1.0:1775776605.500270:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308160524 00000100:00000001:1.0:1775776605.500272:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308164358 00000100:00000001:1.0:1775776605.500274:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776605.500276:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776605.500278:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776605.500281:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776605/real 1775776605] req@ffff8f6202c71180 x1862022710427008/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776605.500288:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776605.500290:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776605.500291:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776605.500294:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776605.500295:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776605.500297:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776605.500300:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71180 x1862022710427008/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776605.500307:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71180 x1862022710427008/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.500313:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776605.500314:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776605.500316:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776605.500317:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776605.500321:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71180 x1862022710427008/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.500327:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776605.500330:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71180 x1862022710427008/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.500336:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776605.500339:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776605.500340:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776605.500341:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776605.500343:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776605.500348:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776605.500350:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776605.500353:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776605.500355:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776605.500359:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71180 x1862022710427008/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776605.500366:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710427008:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776605.500369:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776605.500372:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72300 x1862022710427136/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.500378:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776605.500380:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776605.500382:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776605.500383:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776605.500385:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776605.500388:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710427136:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776605.500391:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776605.500392:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776605.500394:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776605.500396:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776605.500398:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776605.500400:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776605.500403:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776605.500405:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776605.500407:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776605.500410:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710427136, portal 10 00000100:00000001:1.0:1775776605.500412:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776605.500413:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849012992 : -123823860538624 : ffff8f6202c72300) 00000100:00000040:1.0:1775776605.500417:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72300 x1862022710427136/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776630 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.500423:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776605.500426:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776605.500427:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776605.500430:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710427136, offset 0 00000400:00000200:1.0:1775776605.500433:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776605.500436:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776605.500439:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776605.500442:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776605.500444:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776605.500446:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776605.500448:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776605.500451:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453889:453889:256:4294967295] 192.168.201.115@tcp LPNI seq info [453814:453814:8:4294967295] 00000400:00000200:1.0:1775776605.500455:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776605.500461:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776605.500465:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776605.500467:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776605.500471:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776605.500473:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1775776605.500476:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776605.500479:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 37 secs later) 00000800:00000100:1.0:1775776605.500483:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776605.500485:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776605.500487:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776605.500490:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776605.500492:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776605.500494:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776605.500496:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776605.500499:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72300 x1862022710427136/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776630 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.500507:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776605.500509:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72300 x1862022710427136/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776630 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.500515:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776605.500517:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776605.500519:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776605.500522:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776605.500525:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776605.500526:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308160524 00000100:00000001:1.0:1775776605.500529:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308164358 00000100:00000001:1.0:1775776605.500530:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308160524 00000100:00000001:1.0:1775776605.500532:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308164358 00000100:00000001:1.0:1775776605.500534:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776605.500536:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776605.500538:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776605.500541:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776605/real 1775776605] req@ffff8f6202c72300 x1862022710427136/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776605.500547:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776605.500549:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776605.500550:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776605.500552:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776605.500553:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776605.500555:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776605.500558:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72300 x1862022710427136/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776605.500565:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72300 x1862022710427136/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.500572:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776605.500573:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776605.500575:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776605.500576:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776605.500579:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72300 x1862022710427136/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.500586:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776605.500589:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72300 x1862022710427136/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.500608:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776605.500610:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776605.500612:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776605.500613:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776605.500614:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776605.500616:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776605.500618:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776605.500621:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776605.500623:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776605.500627:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72300 x1862022710427136/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776605.500634:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710427136:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776605.500638:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776605.500639:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776605.500642:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71180 x1862022710427008/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.500649:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776605.500650:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776605.500652:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776605.500654:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776605.500655:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776605.500657:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776605.500658:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776605.500660:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34800. 02000000:00000001:1.0:1775776605.500662:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776605.500663:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776605.500665:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71180. 00000100:00000001:1.0:1775776605.500667:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776605.500668:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776605.500670:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776605.500672:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72300 x1862022710427136/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776630 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776605.500678:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776605.500680:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776605.500681:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776605.500683:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776605.500684:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776605.500685:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776605.500686:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776605.500687:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34400. 02000000:00000001:1.0:1775776605.500689:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776605.500690:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776605.500691:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72300. 00000100:00000001:1.0:1775776605.500693:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776605.500694:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776605.500695:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776605.500698:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776605.500699:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776605.500701:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776605.500702:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776605.627387:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776605.627392:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776605.627398:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776605.627399:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776605.627403:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776605.627404:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776605.628786:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776605.628790:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776605.628794:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776606.523265:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776606.523273:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776606.523275:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776606.523284:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776606.523288:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776606.523291:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776606.523294:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776606.523297:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776606.523299:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776606.651266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776606.651273:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776606.651275:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776606.651279:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776606.651283:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776606.651285:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776606.651287:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776606.651289:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776606.651292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776606.651294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776606.651297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776606.651298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776606.651369:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776606.651372:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776606.651375:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776607.547237:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776607.547244:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776607.547246:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776607.547252:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776607.547255:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776607.547259:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776607.547262:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776607.547265:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776607.547267:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776607.675259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776607.675264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776607.675269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776607.675270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776607.675274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776607.675275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776607.675338:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776607.675342:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776607.675346:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776608.571248:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776608.571255:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776608.571257:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776608.571262:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776608.571266:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776608.571269:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776608.571272:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776608.571274:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776608.571276:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776608.699356:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776608.699362:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776608.699367:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776608.699368:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776608.699371:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776608.699372:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776608.699480:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776608.699486:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776608.699491:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776609.403260:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308164358, at 4308164416 00000100:00000040:0.0:1775776609.403269:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308164358, at 4308164416 00000100:00000010:1.0:1775776609.403282:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2c60. 00000100:00000010:0.0:1775776609.403285:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ae0. 00000100:00000001:0.0:1775776609.595301:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776609.595309:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776609.595311:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776609.595318:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776609.595323:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776609.595327:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776609.595330:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776609.595333:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776609.595336:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776609.723270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776609.723278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776609.723289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776609.723291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776609.723296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776609.723297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776609.723356:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776609.723359:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776609.723362:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776610.619265:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776610.619271:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776610.619273:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776610.619278:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776610.619281:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776610.619284:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776610.619287:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776610.619289:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776610.619292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776610.619311:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776610.619315:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776610.619318:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776610.619321:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776610.619324:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776610.619332:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13493 00000400:00000200:0.0:1775776610.619341:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776610.619346:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776610.619349:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776610.619352:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776610.619364:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776610.619366:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776610.619368:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776610.619372:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776610.619374:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776610.619377:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776610.619379:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776610.619382:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776610.619384:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776610.619387:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776610.619389:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776610.619391:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776610.619393:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776610.619396:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776610.619399:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776610.619401:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776610.619405:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776610.619408:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776610.619411:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776610.619498:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776610.619501:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776610.619532:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063b80. 00000020:00000040:0.0:1775776610.619536:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776610.619540:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776610.619542:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776610.619544:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776610.619546:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776610.619551:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776610.619558:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7000. 00000100:00000001:0.0:1775776610.619564:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776610.619573:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063b80 x1862022710427264/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776610.619584:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776610.619588:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776610.619592:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ee9007b] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063b80 x1862022710427264/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776610.619616:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776610.619619:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776610.619623:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776610.619624:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776610.619626:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776610.619628:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776610.619630:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13493 00000400:00000200:0.0:1775776610.619635:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776610.619638:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776610.619641:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776610.619644:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776610.619647:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776610.619649:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776610.619652:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776610.619654:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1775776610.619654:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:0.0:1775776610.619658:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000001:1.0:1775776610.619659:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1775776610.619660:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776610.619663:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:1.0:1775776610.619664:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1775776610.619667:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775776610.619669:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775776610.619672:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776610.619675:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776610.619678:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776610.619681:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:1.0:1775776610.619682:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e063b80 x1862022710427264/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1775776610.619685:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775776610.619691:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776610.619698:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776610.619699:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1775776610.619701:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776610.619703:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776610.619707:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776610.619707:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776610.619711:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776610.619712:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776610.619714:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776610.619717:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1775776610.619720:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775776610.619722:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e063b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710427264:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776610.619728:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000008:00000001:0.0:1775776610.619731:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776610.619732:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1775776610.619734:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776610.619735:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776610.619739:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061f80. 02000000:00000001:1.0:1775776610.619739:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:0.0:1775776610.619743:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776610.619746:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000010:1.0:1775776610.619746:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d400. 02000000:00000001:0.0:1775776610.619749:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776610.619750:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776610.619752:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776610.619754:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776610.619756:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776610.619761:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7c00. 00000400:00000200:1.0:1775776610.619762:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776610.619767:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776610.619767:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776610.619774:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776610.619780:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710427264, portal 25 00000100:00000001:1.0:1775776610.619781:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776610.619782:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574568320 : -123823134983296 : ffff8f622e063b80) 00000100:00100000:0.0:1775776610.619790:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061f80 x1862022710427392/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1775776610.619790:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e063b80 x1862022710427264/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776635 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776610.619807:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776610.619812:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:0.0:1775776610.619813:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1775776610.619814:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776610.619817:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710427264, offset 0 00000100:00000001:0.0:1775776610.619821:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776610.619823:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000040:0.0:1775776610.619829:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000029d3794] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061f80 x1862022710427392/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776610.619831:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776610.619841:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776610.619845:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776610.619846:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00080000:0.0:1775776610.619852:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:1.0:1775776610.619853:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1775776610.619856:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776610.619859:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1775776610.619859:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:0.0:1775776610.619864:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:1.0:1775776610.619866:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00080000:0.0:1775776610.619870:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13493 00000400:00000200:1.0:1775776610.619872:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776610.619879:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453890:453890:256:4294967295] 192.168.201.115@tcp LPNI seq info [453815:453815:8:4294967295] 00000400:00000200:1.0:1775776610.619884:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776610.619894:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776610.619899:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776610.619903:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000010:1.0:1775776610.619905:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000400:00000200:0.0:1775776610.619908:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776610.619911:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1775776610.619913:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776610.619916:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000100:00000040:0.0:1775776610.619917:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776610.619922:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1775776610.619925:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:0.0:1775776610.619928:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:1.1:1775776610.619929:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000100:00000001:0.0:1775776610.619933:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776610.619936:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776610.619937:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776610.619939:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776610.619941:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000800:00000100:1.0:1775776610.619944:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000020:00000001:0.0:1775776610.619945:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000800:00000010:1.0:1775776610.619948:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000020:00000001:0.0:1775776610.619949:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775776610.619955:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776610.619956:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1775776610.619959:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776610.619964:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776610.619966:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1775776610.619969:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776610.619972:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776610.619972:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776610.619976:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00080000:0.0:1775776610.619977:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776610.619982:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776610.619984:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776610.619984:0:2411:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1775776610.619987:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776610.619992:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1775776610.619992:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e063b80 x1862022710427264/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776635 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776610.619997:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776610.620003:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776610.620005:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e063b80 x1862022710427264/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776635 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775776610.620007:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776610.620013:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776610.620014:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776610.620019:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776610.620019:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:0.0:1775776610.620024:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061180. 00000100:00000001:1.0:1775776610.620024:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1775776610.620029:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:1.0:1775776610.620030:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:0.0:1775776610.620034:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1775776610.620034:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308169478 02000000:00000001:0.0:1775776610.620037:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776610.620039:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000010:1.0:1775776610.620039:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2c60. 02000000:00000001:0.0:1775776610.620041:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776610.620044:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776610.620046:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 02000000:00000010:0.0:1775776610.620050:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5c00. 00000100:00000001:1.0:1775776610.620051:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308169478 00000100:00000001:0.0:1775776610.620056:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776610.620056:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2540. 00000100:00000001:1.0:1775776610.620061:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00100000:0.0:1775776610.620065:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061180 x1862022710427520/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776610.620065:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776610.620068:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776610.620071:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776610.620076:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776610/real 1775776610] req@ffff8f622e063b80 x1862022710427264/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776610.620080:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776610.620085:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776610.620093:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e23859] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061180 x1862022710427520/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776610.620101:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776610.620103:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1775776610.620106:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776610.620107:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000100:00000040:0.0:1775776610.620110:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776610.620111:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776610.620115:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000040:0.0:1775776610.620117:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776610.620119:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:0.0:1775776610.620122:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1775776610.620126:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e063b80 x1862022710427264/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776610.620128:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000200:1.0:1775776610.620160:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e063b80 x1862022710427264/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1775776610.620162:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13503) 00000100:00000001:1.0:1775776610.620167:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776610.620169:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776610.620171:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776610.620173:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776610.620178:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e063b80 x1862022710427264/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776610.620185:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776610.620190:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e063b80 x1862022710427264/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776610.620196:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776610.620199:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776610.620201:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776610.620202:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776610.620204:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776610.620206:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776610.620209:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776610.620213:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776610.620215:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776610.620219:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e063b80 x1862022710427264/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776610.620228:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e063b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710427264:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776610.620232:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776610.620235:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776610.620237:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e063b80 x1862022710427264/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776610.620242:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776610.620243:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776610.620245:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776610.620248:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776610.620249:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776610.620250:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776610.620251:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776610.620253:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e7000. 02000000:00000001:1.0:1775776610.620255:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776610.620257:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776610.620258:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e063b80. 00000100:00000001:1.0:1775776610.620260:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776610.620261:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776610.620262:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776610.620267:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776610.620268:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776610.620269:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776610.620270:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776610.620271:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776610.620273:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061f80 x1862022710427392/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776610.620277:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776610.620278:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776610.620280:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776610.620281:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776610.620282:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776610.620284:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710427392:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776610.620286:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776610.620288:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776610.620289:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776610.620290:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776610.620292:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776610.620293:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776610.620297:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776610.620298:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776610.620301:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776610.620303:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710427392, portal 10 00000100:00000001:1.0:1775776610.620305:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776610.620306:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574561152 : -123823134990464 : ffff8f622e061f80) 00000100:00000040:1.0:1775776610.620311:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061f80 x1862022710427392/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776635 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776610.620318:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776610.620320:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776610.620322:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776610.620324:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710427392, offset 0 00000400:00000200:1.0:1775776610.620329:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776610.620332:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776610.620337:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776610.620340:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776610.620342:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776610.620345:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776610.620347:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776610.620358:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453891:453891:256:4294967295] 192.168.201.115@tcp LPNI seq info [453816:453816:8:4294967295] 00000400:00000200:1.0:1775776610.620363:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776610.620369:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776610.620373:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776610.620377:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776610.620380:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776610.620383:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775776610.620387:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776610.620390:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1775776610.620393:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776610.620395:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776610.620397:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776610.620400:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776610.620403:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776610.620405:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776610.620407:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776610.620411:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061f80 x1862022710427392/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776635 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776610.620418:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776610.620421:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061f80 x1862022710427392/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776635 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776610.620427:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776610.620429:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776610.620432:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776610.620435:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776610.620438:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776610.620440:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308165644 00000100:00000001:1.0:1775776610.620442:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308169478 00000100:00000001:1.0:1775776610.620444:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308165644 00000100:00000001:1.0:1775776610.620447:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308169478 00000100:00000001:1.0:1775776610.620449:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776610.620451:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776610.620452:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776610.620456:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776610/real 1775776610] req@ffff8f622e061f80 x1862022710427392/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776610.620463:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776610.620464:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776610.620466:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776610.620468:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776610.620469:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776610.620471:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776610.620473:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061f80 x1862022710427392/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776610.620477:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061f80 x1862022710427392/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776610.620481:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776610.620482:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776610.620483:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776610.620484:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776610.620487:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061f80 x1862022710427392/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776610.620493:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776610.620496:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061f80 x1862022710427392/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776610.620515:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776610.620518:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776610.620519:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776610.620520:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776610.620522:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776610.620526:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776610.620529:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776610.620532:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776610.620535:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776610.620539:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061f80 x1862022710427392/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776610.620546:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710427392:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776610.620550:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776610.620553:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061180 x1862022710427520/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776610.620560:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776610.620561:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776610.620563:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776610.620565:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776610.620567:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776610.620570:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710427520:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776610.620573:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776610.620575:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776610.620576:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776610.620578:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776610.620580:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776610.620582:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776610.620584:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776610.620586:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776610.620588:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776610.620590:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710427520, portal 10 00000100:00000001:1.0:1775776610.620591:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776610.620592:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574557568 : -123823134994048 : ffff8f622e061180) 00000100:00000040:1.0:1775776610.620596:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061180 x1862022710427520/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776635 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776610.620600:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776610.620601:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776610.620603:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776610.620604:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710427520, offset 0 00000400:00000200:1.0:1775776610.620607:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776610.620610:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776610.620613:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776610.620616:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776610.620618:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776610.620621:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776610.620623:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776610.620626:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453892:453892:256:4294967295] 192.168.201.115@tcp LPNI seq info [453817:453817:8:4294967295] 00000400:00000200:1.0:1775776610.620629:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776610.620636:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776610.620640:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776610.620643:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776610.620647:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776610.620650:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1775776610.620654:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776610.620657:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1775776610.620660:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776610.620662:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776610.620664:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776610.620668:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776610.620671:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776610.620672:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776610.620674:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776610.620677:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061180 x1862022710427520/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776635 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776610.620684:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776610.620687:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061180 x1862022710427520/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776635 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776610.620693:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776610.620694:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776610.620697:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776610.620700:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776610.620703:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776610.620705:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308165644 00000100:00000001:1.0:1775776610.620707:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308169478 00000100:00000001:1.0:1775776610.620709:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308165644 00000100:00000001:1.0:1775776610.620711:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308169478 00000100:00000001:1.0:1775776610.620713:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776610.620715:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776610.620717:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776610.620720:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776610/real 1775776610] req@ffff8f622e061180 x1862022710427520/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776610.620728:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776610.620729:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776610.620731:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776610.620733:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776610.620734:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776610.620736:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776610.620739:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061180 x1862022710427520/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776610.620745:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061180 x1862022710427520/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776610.620751:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776610.620752:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776610.620755:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776610.620756:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776610.620759:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061180 x1862022710427520/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776610.620765:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776610.620769:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061180 x1862022710427520/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776610.620775:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776610.620778:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776610.620779:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776610.620780:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776610.620783:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776610.620784:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776610.620787:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776610.620789:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776610.620791:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776610.620795:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061180 x1862022710427520/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776610.620803:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710427520:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776610.620806:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776610.620808:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776610.620811:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061f80 x1862022710427392/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776610.620817:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776610.620819:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776610.620820:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776610.620823:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776610.620824:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776610.620825:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776610.620827:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776610.620829:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e7c00. 02000000:00000001:1.0:1775776610.620831:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776610.620832:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776610.620834:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061f80. 00000100:00000001:1.0:1775776610.620836:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776610.620837:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776610.620839:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776610.620842:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061180 x1862022710427520/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776635 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776610.620848:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776610.620849:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776610.620850:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776610.620853:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776610.620854:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776610.620855:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776610.620856:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776610.620858:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e5c00. 02000000:00000001:1.0:1775776610.620860:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776610.620861:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776610.620862:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061180. 00000100:00000001:1.0:1775776610.620864:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776610.620865:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776610.620867:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776610.620870:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776610.620871:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776610.620873:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776610.620874:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776610.747252:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776610.747256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776610.747259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776610.747261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776610.747264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776610.747265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776610.747331:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776610.747334:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776610.747337:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776611.643254:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776611.643262:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776611.643264:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776611.643269:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776611.643273:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776611.643277:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776611.643280:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776611.643282:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776611.643284:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776611.771265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776611.771270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776611.771275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776611.771278:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776611.771281:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776611.771284:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776611.771287:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776611.771289:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776611.771290:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776611.771293:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776611.771297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776611.771299:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776611.771300:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776611.771301:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776611.771303:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776611.771304:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776611.771305:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776611.771307:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776611.771480:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776611.771505:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776611.771507:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776612.667264:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776612.667278:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776612.667280:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776612.667287:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776612.667291:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776612.667294:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776612.667297:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776612.667299:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776612.667302:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776612.795275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776612.795282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776612.795289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776612.795290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776612.795294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776612.795296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776612.795378:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776612.795384:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776612.795388:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776613.691207:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776613.691212:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776613.691214:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776613.691218:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776613.691221:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776613.691223:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776613.691225:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776613.691226:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776613.691229:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776613.819578:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776613.819585:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776613.819590:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776613.819592:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776613.819596:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776613.819598:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776613.819667:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776613.819671:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776613.819675:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776614.523238:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308169478, at 4308169536 00000100:00000040:1.0:1775776614.523239:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308169478, at 4308169536 00000100:00000010:0.0:1775776614.523255:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2c60. 00000100:00000010:1.0:1775776614.523256:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2540. 00000100:00000001:0.0:1775776614.715262:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776614.715269:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776614.715271:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776614.715277:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776614.715281:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776614.715284:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776614.715287:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776614.715290:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776614.715292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776614.843247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776614.843252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776614.843255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776614.843256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776614.843259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776614.843260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776614.843319:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776614.843323:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776614.843326:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776615.739271:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776615.739274:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776615.739280:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1775776615.739282:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776615.739283:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776615.739286:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1775776615.739289:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:3.0:1775776615.739291:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776615.739293:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776615.739296:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775776615.739297:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776615.739301:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776615.739305:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1775776615.739305:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13498 00000100:00000001:0.0:1775776615.739309:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776615.739317:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776615.739324:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776615.739327:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776615.739331:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776615.739334:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776615.739337:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776615.739339:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776615.739342:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776615.739343:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776615.739345:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776615.739347:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776615.739348:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776615.739350:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776615.739352:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776615.739353:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776615.739354:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776615.739355:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776615.739357:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776615.739358:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776615.739360:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776615.739362:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776615.739364:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776615.739366:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776615.739367:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776615.739370:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776615.739377:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775776615.739379:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776615.739382:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776615.739383:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776615.739386:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776615.739388:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776615.739393:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776615.739401:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775776615.739407:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776615.739447:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710427648/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776615.739458:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776615.739462:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776615.739467:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710427648/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776615.739483:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776615.739486:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776615.739488:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776615.739489:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776615.739490:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776615.739492:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776615.739493:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13498 00000400:00000200:3.0:1775776615.739496:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776615.739498:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776615.739500:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776615.739503:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776615.739505:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776615.739507:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776615.739508:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776615.739511:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776615.739513:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776615.739515:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776615.739517:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776615.739519:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776615.739521:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776615.739525:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776615.739526:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776615.739528:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776615.739529:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776615.739530:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:1.0:1775776615.739534:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1775776615.739535:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000001:1.0:1775776615.739538:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1775776615.739539:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776615.739543:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776615.739547:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1775776615.739548:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72a00 x1862022710427648/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1775776615.739552:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776615.739558:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776615.739561:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776615.739563:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776615.739565:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775776615.739566:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775776615.739569:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:3.0:1775776615.739571:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776615.739573:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1775776615.739573:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776615.739577:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 02000000:00000001:1.0:1775776615.739578:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776615.739581:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000100:00100000:1.0:1775776615.739585:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710427648:192.168.201.115@tcp:250:kworker.0 02000000:00000001:3.0:1775776615.739586:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776615.739588:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776615.739592:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1775776615.739592:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776615.739593:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776615.739597:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776615.739598:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776615.739601:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776615.739601:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 02000000:00000001:1.0:1775776615.739604:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775776615.739606:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776615.739609:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776615.739612:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776615.739614:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710427776/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776615.739619:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776615.739625:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000001:3.0:1775776615.739627:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1775776615.739630:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000001:3.0:1775776615.739630:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776615.739637:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710427776/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776615.739638:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710427648, portal 25 00000100:00000001:1.0:1775776615.739642:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776615.739644:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849014784 : -123823860536832 : ffff8f6202c72a00) 00000100:00000001:3.0:1775776615.739652:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776615.739653:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72a00 x1862022710427648/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776640 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775776615.739655:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776615.739662:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776615.739665:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1775776615.739668:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1775776615.739668:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776615.739672:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000040:1.0:1775776615.739673:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776615.739676:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00080000:3.0:1775776615.739676:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13498 00000100:00000200:1.0:1775776615.739680:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710427648, offset 0 00000400:00000200:3.0:1775776615.739683:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776615.739686:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000400:00000200:1.0:1775776615.739690:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00080000:3.0:1775776615.739691:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776615.739695:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776615.739697:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776615.739700:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775776615.739701:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776615.739705:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776615.739707:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776615.739707:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000400:00000200:1.0:1775776615.739709:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000020:00000001:3.0:1775776615.739709:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776615.739713:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776615.739714:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776615.739716:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1775776615.739717:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776615.739721:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776615.739723:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1775776615.739724:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000040:3.0:1775776615.739726:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776615.739728:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776615.739729:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776615.739731:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00080000:3.0:1775776615.739734:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776615.739736:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776615.739738:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 02000000:00000001:3.0:1775776615.739739:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776615.739743:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776615.739746:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776615.739747:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453893:453893:256:4294967295] 192.168.201.115@tcp LPNI seq info [453818:453818:8:4294967295] 00000100:00000001:3.0:1775776615.739749:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776615.739751:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776615.739755:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776615.739758:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776615.739759:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776615.739764:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 00000020:00000040:3.0:1775776615.739766:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776615.739769:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776615.739772:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776615.739775:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1775776615.739776:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 02000000:00000001:3.0:1775776615.739777:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776615.739780:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776615.739784:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000800:00000200:1.0:1775776615.739786:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776615.739788:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775776615.739792:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000100:00100000:3.0:1775776615.739794:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710427904/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1775776615.739804:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775776615.739807:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776615.739810:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776615.739811:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000100:00000040:3.0:1775776615.739817:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710427904/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1775776615.739821:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776615.739827:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000100:00000001:3.0:1775776615.739832:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1775776615.739836:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776615.739837:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:1.0:1775776615.739841:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000100:00000040:3.0:1775776615.739843:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776615.739846:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1775776615.739848:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:3.0:1775776615.739853:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1775776615.739857:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000040:3.0:1775776615.739860:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13508) 00000400:00000200:1.0:1775776615.739861:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776615.739864:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776615.739869:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776615.739873:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72a00 x1862022710427648/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776640 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.739880:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776615.739882:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72a00 x1862022710427648/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776640 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.739887:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776615.739888:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776615.739892:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776615.739895:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776615.739897:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308174598 00000100:00000010:1.0:1775776615.739901:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2540. 00000100:00000001:1.0:1775776615.739905:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776615.739910:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308174598 00000100:00000010:1.0:1775776615.739913:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2c60. 00000100:00000001:1.0:1775776615.739915:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776615.739917:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776615.739919:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776615.739920:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776615.739923:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776615/real 1775776615] req@ffff8f6202c72a00 x1862022710427648/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776615.739930:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776615.739931:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776615.739932:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776615.739933:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776615.739934:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776615.739936:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776615.739938:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72a00 x1862022710427648/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776615.739943:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72a00 x1862022710427648/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.739947:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776615.739948:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776615.739949:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776615.739951:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776615.739953:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72a00 x1862022710427648/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.739957:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776615.739959:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72a00 x1862022710427648/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.739963:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776615.739965:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776615.739965:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776615.739966:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776615.739968:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776615.739969:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776615.739970:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776615.739973:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776615.739974:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776615.739976:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72a00 x1862022710427648/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776615.739981:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710427648:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776615.739984:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776615.739985:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776615.739987:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72a00 x1862022710427648/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.739994:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776615.739996:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776615.739998:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776615.740000:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776615.740001:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776615.740003:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776615.740005:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776615.740007:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34c00. 02000000:00000001:1.0:1775776615.740010:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776615.740012:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776615.740013:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72a00. 00000100:00000001:1.0:1775776615.740016:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776615.740017:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776615.740019:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776615.740025:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776615.740026:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776615.740028:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776615.740029:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776615.740031:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776615.740034:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70700 x1862022710427776/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.740040:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776615.740049:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776615.740052:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776615.740053:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776615.740056:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776615.740058:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710427776:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776615.740062:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776615.740064:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776615.740065:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776615.740067:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776615.740069:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776615.740071:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776615.740073:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776615.740076:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776615.740078:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776615.740080:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710427776, portal 10 00000100:00000001:1.0:1775776615.740081:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776615.740082:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849005824 : -123823860545792 : ffff8f6202c70700) 00000100:00000040:1.0:1775776615.740087:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70700 x1862022710427776/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776640 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.740092:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776615.740094:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776615.740096:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776615.740099:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710427776, offset 0 00000400:00000200:1.0:1775776615.740102:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776615.740105:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776615.740109:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776615.740111:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776615.740114:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776615.740116:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776615.740118:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776615.740121:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453894:453894:256:4294967295] 192.168.201.115@tcp LPNI seq info [453819:453819:8:4294967295] 00000400:00000200:1.0:1775776615.740125:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776615.740130:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776615.740161:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776615.740164:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776615.740168:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776615.740170:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775776615.740174:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776615.740176:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775776615.740179:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776615.740181:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776615.740183:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776615.740186:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776615.740189:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776615.740191:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776615.740194:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776615.740197:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70700 x1862022710427776/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776640 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.740204:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776615.740207:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70700 x1862022710427776/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776640 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.740213:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776615.740215:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776615.740218:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776615.740222:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776615.740225:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776615.740226:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308170764 00000100:00000001:1.0:1775776615.740229:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308174598 00000100:00000001:1.0:1775776615.740231:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308170764 00000100:00000001:1.0:1775776615.740232:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308174598 00000100:00000001:1.0:1775776615.740234:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776615.740235:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776615.740236:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776615.740238:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776615/real 1775776615] req@ffff8f6202c70700 x1862022710427776/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776615.740243:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776615.740244:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776615.740245:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776615.740247:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776615.740248:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776615.740249:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776615.740251:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70700 x1862022710427776/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776615.740255:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70700 x1862022710427776/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.740263:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776615.740264:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776615.740272:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776615.740273:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776615.740275:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70700 x1862022710427776/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.740279:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776615.740282:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70700 x1862022710427776/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.740289:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776615.740291:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776615.740293:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776615.740294:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776615.740296:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776615.740301:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776615.740303:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776615.740306:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776615.740309:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776615.740312:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70700 x1862022710427776/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776615.740320:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710427776:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776615.740324:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776615.740326:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70a80 x1862022710427904/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.740330:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776615.740331:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776615.740333:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776615.740334:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776615.740335:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776615.740337:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710427904:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776615.740339:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776615.740340:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776615.740341:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776615.740342:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776615.740345:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776615.740346:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776615.740349:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776615.740351:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776615.740353:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776615.740355:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710427904, portal 10 00000100:00000001:1.0:1775776615.740356:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776615.740357:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849006720 : -123823860544896 : ffff8f6202c70a80) 00000100:00000040:1.0:1775776615.740362:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70a80 x1862022710427904/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776640 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.740366:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776615.740367:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776615.740369:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776615.740371:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710427904, offset 0 00000400:00000200:1.0:1775776615.740375:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776615.740379:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776615.740382:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776615.740385:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776615.740387:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776615.740389:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776615.740391:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776615.740394:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453895:453895:256:4294967295] 192.168.201.115@tcp LPNI seq info [453820:453820:8:4294967295] 00000400:00000200:1.0:1775776615.740398:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776615.740404:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776615.740409:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776615.740469:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776615.740475:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776615.740477:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775776615.740481:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776615.740484:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775776615.740488:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776615.740489:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776615.740492:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776615.740495:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776615.740498:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776615.740499:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776615.740501:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776615.740503:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70a80 x1862022710427904/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776640 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.740508:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776615.740510:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70a80 x1862022710427904/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776640 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.740513:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776615.740514:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776615.740516:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776615.740518:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776615.740521:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776615.740523:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308170764 00000100:00000001:1.0:1775776615.740525:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308174598 00000100:00000001:1.0:1775776615.740527:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308170764 00000100:00000001:1.0:1775776615.740529:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308174598 00000100:00000001:1.0:1775776615.740531:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776615.740532:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776615.740534:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776615.740537:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776615/real 1775776615] req@ffff8f6202c70a80 x1862022710427904/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776615.740544:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776615.740546:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776615.740547:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776615.740549:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776615.740551:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776615.740553:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776615.740555:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70a80 x1862022710427904/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776615.740562:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70a80 x1862022710427904/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.740568:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776615.740569:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776615.740571:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776615.740572:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776615.740575:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70a80 x1862022710427904/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.740581:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776615.740585:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70a80 x1862022710427904/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.740591:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776615.740593:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776615.740595:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776615.740596:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776615.740597:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776615.740599:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776615.740601:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776615.740604:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776615.740606:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776615.740610:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70a80 x1862022710427904/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776615.740615:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710427904:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776615.740617:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776615.740618:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776615.740620:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70700 x1862022710427776/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.740623:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776615.740624:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776615.740626:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d400. 02000000:00000001:1.0:1775776615.740627:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776615.740628:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776615.740629:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776615.740630:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776615.740632:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36000. 02000000:00000001:1.0:1775776615.740633:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776615.740634:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776615.740635:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70700. 00000100:00000001:1.0:1775776615.740637:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776615.740637:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776615.740638:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776615.740640:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70a80 x1862022710427904/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776640 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776615.740643:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776615.740644:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776615.740644:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:1.0:1775776615.740645:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776615.740646:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776615.740647:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776615.740647:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776615.740648:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36400. 02000000:00000001:1.0:1775776615.740649:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776615.740650:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776615.740651:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70a80. 00000100:00000001:1.0:1775776615.740652:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776615.740653:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776615.740654:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776615.740657:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776615.740657:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776615.740659:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776615.740660:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776615.867244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776615.867249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776615.867253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776615.867254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776615.867256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776615.867257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776615.867312:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776615.867315:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776615.867318:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776616.763210:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776616.763215:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776616.763217:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776616.763221:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776616.763224:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776616.763226:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776616.763228:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776616.763230:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776616.763232:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776616.891292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776616.891298:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776616.891300:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776616.891304:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776616.891308:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776616.891310:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776616.891312:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776616.891314:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776616.891318:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776616.891319:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776616.891323:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776616.891324:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776616.891428:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776616.891432:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776616.891435:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776617.787273:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776617.787280:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776617.787282:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776617.787288:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776617.787291:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776617.787295:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776617.787297:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776617.787300:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776617.787303:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776617.915219:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776617.915226:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776617.915231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776617.915232:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776617.915234:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776617.915235:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776617.915317:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776617.915323:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776617.915328:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776618.811264:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776618.811270:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776618.811271:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776618.811276:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776618.811280:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776618.811283:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776618.811285:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776618.811287:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776618.811289:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776618.939297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776618.939304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776618.939310:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776618.939311:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776618.939315:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776618.939317:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776618.939438:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776618.939442:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776618.939445:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776619.643279:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308174598, at 4308174656 00000100:00000040:1.0:1775776619.643279:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308174598, at 4308174656 00000100:00000010:0.0:1775776619.643303:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2540. 00000100:00000010:1.0:1775776619.643306:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2c60. 00000100:00000001:0.0:1775776619.835285:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776619.835294:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776619.835297:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776619.835303:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776619.835308:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776619.835313:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776619.835316:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776619.835319:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776619.835322:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776619.963209:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776619.963216:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776619.963221:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776619.963223:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776619.963227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776619.963228:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776619.963679:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776619.963685:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776619.963689:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776620.859267:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776620.859274:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776620.859277:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776620.859283:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:1.0:1775776620.859286:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776620.859288:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776620.859294:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775776620.859295:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776620.859302:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776620.859302:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775776620.859309:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1775776620.859311:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776620.859315:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776620.859318:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776620.859343:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13503 00000400:00000200:1.0:1775776620.859357:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776620.859366:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776620.859370:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1775776620.859374:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776620.859379:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776620.859382:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776620.859384:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:1.0:1775776620.859388:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776620.859390:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:1.0:1775776620.859392:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776620.859395:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:1.0:1775776620.859397:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:1.0:1775776620.859399:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:1.0:1775776620.859401:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776620.859402:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776620.859403:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776620.859405:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:1.0:1775776620.859408:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:1.0:1775776620.859411:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776620.859413:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776620.859417:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1775776620.859421:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776620.859424:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776620.859427:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776620.859429:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776620.859441:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623491a680. 00000020:00000040:1.0:1775776620.859445:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:1.0:1775776620.859449:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776620.859451:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776620.859455:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776620.859458:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776620.859462:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776620.859472:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4ec00. 00000100:00000001:1.0:1775776620.859480:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776620.859491:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623491a680 x1862022710428032/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1775776620.859502:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776620.859507:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776620.859512:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029bc1460] to pc [ptlrpcd_rcv+-1] req@ffff8f623491a680 x1862022710428032/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776620.859540:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1775776620.859543:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776620.859547:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1775776620.859548:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776620.859550:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776620.859554:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1775776620.859557:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13503 00000400:00000200:1.0:1775776620.859563:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776620.859567:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:0.0:1775776620.859570:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1775776620.859570:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775776620.859576:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1775776620.859577:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776620.859582:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1775776620.859583:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776620.859588:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776620.859592:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:0.0:1775776620.859596:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491a680 x1862022710428032/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1775776620.859597:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776620.859600:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:1.0:1775776620.859607:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1775776620.859614:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:1.0:1775776620.859622:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775776620.859623:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776620.859627:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776620.859629:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 02000000:00000001:0.0:1775776620.859632:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1775776620.859637:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776620.859640:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776620.859643:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:0.0:1775776620.859645:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776620.859645:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:1.0:1775776620.859648:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:1.0:1775776620.859654:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000001:0.0:1775776620.859657:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776620.859658:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776620.859665:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:0.0:1775776620.859669:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710428032:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776620.859675:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776620.859675:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776620.859681:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776620.859682:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1775776620.859684:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775776620.859687:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1775776620.859691:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1775776620.859692:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:1.0:1775776620.859699:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776620.859701:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776620.859704:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776620.859708:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776620.859709:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6234919c00. 00000020:00000040:1.0:1775776620.859716:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776620.859721:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1775776620.859723:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:1.0:1775776620.859725:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776620.859729:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776620.859730:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:0.0:1775776620.859731:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d68. 02000000:00000001:1.0:1775776620.859734:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1775776620.859739:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4d400. 00000400:00000010:0.0:1775776620.859742:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000001:1.0:1775776620.859744:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776620.859754:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6234919c00 x1862022710428160/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1775776620.859755:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710428032, portal 25 00000100:00000001:0.0:1775776620.859764:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776620.859770:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684368512 : -123823025183104 : ffff8f623491a680) 00000100:00000001:1.0:1775776620.859782:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:0.0:1775776620.859786:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491a680 x1862022710428032/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776645 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776620.859793:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776620.859807:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000718661bc] to pc [ptlrpcd_rcv+-1] req@ffff8f6234919c00 x1862022710428160/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.859808:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776620.859817:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776620.859825:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0908. 00000100:00000200:0.0:1775776620.859833:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710428032, offset 0 00000100:00000001:1.0:1775776620.859847:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776620.859851:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00080000:1.0:1775776620.859856:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775776620.859862:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:0.0:1775776620.859867:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776620.859868:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1775776620.859877:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1775776620.859883:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:1.0:1775776620.859885:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:0.0:1775776620.859892:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:1.0:1775776620.859893:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13503 00000400:00000200:0.0:1775776620.859902:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776620.859906:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776620.859910:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776620.859913:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453896:453896:256:4294967295] 192.168.201.115@tcp LPNI seq info [453821:453821:8:4294967295] 00000400:00000200:0.0:1775776620.859918:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776620.859926:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776620.859934:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776620.859935:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000010:0.0:1775776620.859942:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111ca00. 00000400:00000200:1.0:1775776620.859945:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:1.0:1775776620.859952:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000800:00000200:0.0:1775776620.859957:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:1.0:1775776620.859960:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000800:00000200:0.0:1775776620.859964:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000100:00000001:1.0:1775776620.859965:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776620.859972:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000800:00000200:0.1:1775776620.859979:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:1.0:1775776620.859980:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000200:0.1:1775776620.859987:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1775776620.859996:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1775776620.859999:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000800:00000100:0.0:1775776620.860000:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000020:00000001:1.0:1775776620.860005:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000010:0.0:1775776620.860006:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111ca00. 00000020:00000040:1.0:1775776620.860009:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:1.0:1775776620.860013:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:0.0:1775776620.860014:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:1.0:1775776620.860018:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:0.0:1775776620.860025:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000040:1.0:1775776620.860030:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:0.0:1775776620.860034:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0908 00000100:00000001:1.0:1775776620.860040:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:0.0:1775776620.860041:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0908. 00000100:00000001:0.0:1775776620.860047:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:1.0:1775776620.860048:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776620.860054:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000200:0.0:1775776620.860055:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491a680 x1862022710428032/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776645 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.860069:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776620.860072:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491a680 x1862022710428032/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776645 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1775776620.860076:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:1.0:1775776620.860087:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:1.0:1775776620.860095:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776620.860096:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776620.860105:0:2411:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1775776620.860106:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775776620.860111:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776620.860112:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776620.860119:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776620.860120:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776620.860124:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776620.860125:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308179718 00000008:00100000:1.0:1775776620.860128:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1775776620.860169:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776620.860172:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1775776620.860177:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623491a300. 00000100:00000010:0.0:1775776620.860179:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c00. 00000020:00000040:1.0:1775776620.860181:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776620.860188:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1775776620.860189:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776620.860192:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776620.860192:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1775776620.860195:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1775776620.860198:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776620.860200:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308179719 00000100:00000010:0.0:1775776620.860208:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c36c0. 02000000:00000010:1.0:1775776620.860208:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622bf4fc00. 00000100:00000001:1.0:1775776620.860217:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776620.860218:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776620.860227:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1775776620.860232:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623491a300 x1862022710428288/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776620.860234:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776620.860244:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776620.860257:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776620/real 1775776620] req@ffff8f623491a680 x1862022710428032/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776620.860277:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776620.860285:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1775776620.860297:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776620.860303:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:1.0:1775776620.860303:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bfa067b] to pc [ptlrpcd_rcv+-1] req@ffff8f623491a300 x1862022710428288/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1775776620.860307:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d68. 00000400:00000200:0.0:1775776620.860312:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776620.860316:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:1.0:1775776620.860338:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776620.860339:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:1.0:1775776620.860343:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:0.0:1775776620.860353:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491a680 x1862022710428032/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776620.860353:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:1.0:1775776620.860360:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1775776620.860373:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000200:0.0:1775776620.860386:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491a680 x1862022710428032/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1775776620.860386:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13513) 00000100:00000001:0.0:1775776620.860401:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776620.860403:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776620.860405:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776620.860406:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776620.860411:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491a680 x1862022710428032/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.860417:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776620.860422:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491a680 x1862022710428032/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.860429:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776620.860431:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776620.860433:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776620.860434:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776620.860436:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776620.860438:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776620.860441:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776620.860446:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776620.860448:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776620.860451:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491a680 x1862022710428032/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776620.860458:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710428032:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776620.860462:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776620.860464:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776620.860467:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491a680 x1862022710428032/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.860479:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776620.860482:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776620.860485:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776620.860488:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776620.860490:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776620.860492:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776620.860494:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776620.860497:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4ec00. 02000000:00000001:0.0:1775776620.860500:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776620.860502:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776620.860504:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491a680. 00000100:00000001:0.0:1775776620.860507:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776620.860508:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776620.860510:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776620.860516:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776620.860518:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776620.860520:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776620.860522:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776620.860523:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776620.860527:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6234919c00 x1862022710428160/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.860534:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776620.860535:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776620.860538:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776620.860539:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776620.860541:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776620.860544:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6234919c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710428160:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776620.860547:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776620.860549:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776620.860551:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776620.860553:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776620.860556:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776620.860558:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776620.860562:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776620.860565:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d68. 00000400:00000010:0.0:1775776620.860568:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776620.860571:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710428160, portal 10 00000100:00000001:0.0:1775776620.860573:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776620.860574:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684365824 : -123823025185792 : ffff8f6234919c00) 00000100:00000040:0.0:1775776620.860580:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6234919c00 x1862022710428160/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776645 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.860586:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776620.860589:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776620.860590:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0908. 00000100:00000200:0.0:1775776620.860593:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710428160, offset 0 00000400:00000200:0.0:1775776620.860596:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776620.860600:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776620.860603:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776620.860606:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776620.860608:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776620.860611:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776620.860613:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776620.860616:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453897:453897:256:4294967295] 192.168.201.115@tcp LPNI seq info [453822:453822:8:4294967295] 00000400:00000200:0.0:1775776620.860620:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776620.860626:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776620.860631:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776620.860633:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111ca00. 00000800:00000200:0.0:1775776620.860637:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776620.860639:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775776620.860644:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776620.860646:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775776620.860650:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776620.860652:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111ca00. 00000400:00000200:0.0:1775776620.860654:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776620.860657:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776620.860660:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0908 00000400:00000010:0.0:1775776620.860662:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0908. 00000100:00000001:0.0:1775776620.860664:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776620.860672:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6234919c00 x1862022710428160/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776645 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.860680:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776620.860683:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6234919c00 x1862022710428160/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776645 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.860690:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776620.860692:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776620.860695:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776620.860698:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776620.860701:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776620.860703:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308175884 00000100:00000001:0.0:1775776620.860706:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308179718 00000100:00000001:0.0:1775776620.860708:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308175884 00000100:00000001:0.0:1775776620.860710:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308179719 00000100:00000001:0.0:1775776620.860712:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776620.860714:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776620.860716:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776620.860719:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776620/real 1775776620] req@ffff8f6234919c00 x1862022710428160/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776620.860726:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776620.860728:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776620.860729:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d68. 00000400:00000200:0.0:1775776620.860731:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776620.860733:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776620.860735:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776620.860738:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6234919c00 x1862022710428160/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776620.860746:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6234919c00 x1862022710428160/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.860758:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776620.860760:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776620.860761:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776620.860763:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776620.860766:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6234919c00 x1862022710428160/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.860772:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776620.860775:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6234919c00 x1862022710428160/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.860781:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776620.860783:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776620.860785:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776620.860786:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776620.860787:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776620.860792:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776620.860794:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776620.860798:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776620.860800:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776620.860803:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6234919c00 x1862022710428160/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776620.860811:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6234919c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710428160:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776620.860814:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776620.860817:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623491a300 x1862022710428288/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.860824:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776620.860825:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776620.860827:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776620.860829:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776620.860831:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776620.860834:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623491a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710428288:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776620.860837:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776620.860838:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776620.860840:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776620.860841:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776620.860844:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776620.860846:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776620.860849:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776620.860851:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d68. 00000400:00000010:0.0:1775776620.860854:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776620.860856:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710428288, portal 10 00000100:00000001:0.0:1775776620.860858:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776620.860859:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250684367616 : -123823025184000 : ffff8f623491a300) 00000100:00000040:0.0:1775776620.860864:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623491a300 x1862022710428288/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776645 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.860870:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776620.860872:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776620.860874:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0908. 00000100:00000200:0.0:1775776620.860876:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710428288, offset 0 00000400:00000200:0.0:1775776620.860880:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776620.860883:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776620.860885:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776620.860888:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776620.860890:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776620.860893:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776620.860895:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776620.860898:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453898:453898:256:4294967295] 192.168.201.115@tcp LPNI seq info [453823:453823:8:4294967295] 00000400:00000200:0.0:1775776620.860902:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776620.860909:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776620.860913:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776620.860916:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111ca00. 00000800:00000200:0.0:1775776620.860920:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776620.860923:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1775776620.860927:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776620.860929:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1775776620.860933:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776620.860934:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111ca00. 00000400:00000200:0.0:1775776620.860937:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776620.860940:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776620.860943:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0908 00000400:00000010:0.0:1775776620.860944:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0908. 00000100:00000001:0.0:1775776620.860947:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776620.860950:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623491a300 x1862022710428288/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776645 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.860957:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776620.860960:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623491a300 x1862022710428288/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776645 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.860966:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776620.860968:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776620.860970:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776620.860972:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776620.860975:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776620.860977:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308175884 00000100:00000001:0.0:1775776620.860979:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308179718 00000100:00000001:0.0:1775776620.860981:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308175884 00000100:00000001:0.0:1775776620.860983:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308179719 00000100:00000001:0.0:1775776620.860985:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776620.860987:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776620.860988:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776620.860991:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776620/real 1775776620] req@ffff8f623491a300 x1862022710428288/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776620.861000:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776620.861001:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776620.861003:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d68. 00000400:00000200:0.0:1775776620.861005:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776620.861006:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776620.861008:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776620.861010:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623491a300 x1862022710428288/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776620.861018:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623491a300 x1862022710428288/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.861024:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776620.861026:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776620.861027:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776620.861029:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776620.861032:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623491a300 x1862022710428288/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.861039:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776620.861042:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623491a300 x1862022710428288/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.861049:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776620.861051:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776620.861052:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776620.861053:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776620.861054:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776620.861056:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776620.861058:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776620.861061:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776620.861063:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776620.861067:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623491a300 x1862022710428288/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776620.861074:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623491a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710428288:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776620.861077:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776620.861079:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776620.861082:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6234919c00 x1862022710428160/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.861088:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776620.861089:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776620.861091:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776620.861093:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776620.861094:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776620.861095:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776620.861097:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776620.861098:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4d400. 02000000:00000001:0.0:1775776620.861100:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776620.861101:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776620.861103:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6234919c00. 00000100:00000001:0.0:1775776620.861105:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776620.861106:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776620.861107:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776620.861110:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623491a300 x1862022710428288/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776645 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776620.861116:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776620.861117:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776620.861119:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776620.861120:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776620.861121:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776620.861122:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776620.861124:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776620.861125:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622bf4fc00. 02000000:00000001:0.0:1775776620.861127:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776620.861129:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776620.861130:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623491a300. 00000100:00000001:0.0:1775776620.861132:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776620.861153:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776620.861155:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776620.861158:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776620.861159:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776620.861161:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776620.861162:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776620.987268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776620.987274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776620.987279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776620.987279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776620.987283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776620.987284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776620.987371:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776620.987375:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776620.987379:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776621.883359:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776621.883368:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776621.883370:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776621.883377:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776621.883382:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776621.883386:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776621.883390:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776621.883393:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776621.883396:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776622.011266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776622.011272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776622.011277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776622.011280:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776622.011282:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776622.011285:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776622.011288:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776622.011290:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776622.011292:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776622.011293:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776622.011296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776622.011297:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776622.011298:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776622.011299:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776622.011299:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776622.011300:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776622.011301:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776622.011317:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776622.011394:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776622.011398:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776622.011401:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776622.907279:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776622.907317:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776622.907319:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776622.907327:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776622.907331:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776622.907336:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776622.907339:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776622.907341:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776622.907344:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776623.035249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776623.035253:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776623.035257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776623.035258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776623.035260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776623.035261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776623.035384:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776623.035388:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776623.035390:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776623.931307:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776623.931314:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776623.931316:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776623.931322:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776623.931326:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776623.931330:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776623.931333:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776623.931335:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776623.931339:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776624.059246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776624.059252:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776624.059257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776624.059258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776624.059262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776624.059263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776624.059355:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776624.059359:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776624.059362:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776624.763267:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308179718, at 4308179776 00000100:00000040:1.0:1775776624.763271:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308179719, at 4308179776 00000100:00000010:0.0:1775776624.763282:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c00. 00000100:00000010:1.0:1775776624.763284:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c36c0. 00000100:00000001:0.0:1775776624.955234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776624.955239:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776624.955241:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776624.955245:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776624.955247:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776624.955250:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776624.955262:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776624.955264:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776624.955266:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776625.083307:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776625.083313:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776625.083318:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776625.083319:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776625.083323:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776625.083324:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776625.083433:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776625.083437:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776625.083440:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776625.979277:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776625.979287:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776625.979290:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776625.979294:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776625.979298:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776625.979299:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776625.979308:0:141703:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13508 00000100:00000001:0.0:1775776625.979309:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776625.979313:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776625.979318:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000400:00000200:3.0:1775776625.979321:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776625.979323:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776625.979327:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1775776625.979330:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:0.0:1775776625.979331:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775776625.979334:0:141703:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775776625.979337:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1775776625.979340:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776625.979341:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776625.979345:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776625.979349:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776625.979352:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776625.979356:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776625.979358:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776625.979361:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776625.979363:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776625.979366:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776625.979369:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776625.979372:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776625.979374:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776625.979376:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776625.979377:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776625.979380:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776625.979383:0:141703:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776625.979386:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776625.979389:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776625.979392:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776625.979395:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776625.979397:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776625.979401:0:141703:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776625.979411:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775776625.979415:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776625.979418:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776625.979420:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776625.979423:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776625.979425:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776625.979429:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776625.979437:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36400. 00000100:00000001:3.0:1775776625.979443:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776625.979452:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710428416/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776625.979462:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776625.979467:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776625.979472:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710428416/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776625.979491:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776625.979494:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776625.979497:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776625.979499:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776625.979500:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776625.979503:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776625.979506:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13508 00000400:00000200:3.0:1775776625.979510:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776625.979514:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776625.979517:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776625.979520:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776625.979521:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776625.979523:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776625.979525:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775776625.979526:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776625.979529:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776625.979530:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776625.979533:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776625.979536:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000100:00000040:0.0:1775776625.979539:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71500 x1862022710428416/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775776625.979539:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776625.979542:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776625.979546:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776625.979552:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775776625.979557:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1775776625.979559:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776625.979560:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776625.979564:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775776625.979567:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1775776625.979567:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776625.979570:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:0.0:1775776625.979571:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1775776625.979573:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:0.0:1775776625.979576:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776625.979578:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776625.979582:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776625.979583:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710428416:192.168.201.115@tcp:250:kworker.0 02000000:00000001:3.0:1775776625.979584:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776625.979592:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776625.979593:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1775776625.979597:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776625.979600:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1775776625.979600:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775776625.979604:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775776625.979605:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1775776625.979608:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:3.0:1775776625.979610:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776625.979612:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776625.979614:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 00000100:00000010:3.0:1775776625.979617:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 02000000:00000001:0.0:1775776625.979618:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1775776625.979621:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776625.979625:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1775776625.979626:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776625.979627:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776625.979631:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776625.979633:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:0.0:1775776625.979634:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d68. 02000000:00000001:3.0:1775776625.979634:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776625.979641:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36000. 00000400:00000010:0.0:1775776625.979643:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000001:3.0:1775776625.979645:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776625.979651:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710428544/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1775776625.979653:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710428416, portal 25 00000100:00000001:0.0:1775776625.979658:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776625.979661:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849009408 : -123823860542208 : ffff8f6202c71500) 00000100:00000001:3.0:1775776625.979667:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776625.979670:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776625.979672:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71500 x1862022710428416/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776650 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776625.979679:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710428544/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.979684:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776625.979691:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776625.979695:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776625.979696:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0908. 00000100:00080000:3.0:1775776625.979699:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:0.0:1775776625.979701:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710428416, offset 0 00000100:00080000:3.0:1775776625.979707:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776625.979708:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776625.979710:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776625.979713:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776625.979716:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13508 00000400:00000200:3.0:1775776625.979719:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776625.979723:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000400:00000200:0.0:1775776625.979725:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00080000:3.0:1775776625.979727:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776625.979731:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776625.979734:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776625.979735:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:0.0:1775776625.979739:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776625.979740:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776625.979744:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776625.979745:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776625.979747:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:0.0:1775776625.979749:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000020:00000040:3.0:1775776625.979750:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776625.979754:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:0.0:1775776625.979757:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1775776625.979757:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776625.979762:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:0.0:1775776625.979765:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:3.0:1775776625.979765:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776625.979770:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:0.0:1775776625.979772:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1775776625.979774:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:0.0:1775776625.979777:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000020:00000040:3.0:1775776625.979779:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776625.979784:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000400:00000200:0.0:1775776625.979786:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453899:453899:256:4294967295] 192.168.201.115@tcp LPNI seq info [453824:453824:8:4294967295] 00000100:00000001:3.0:1775776625.979789:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776625.979793:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776625.979793:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776625.979796:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776625.979799:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776625.979802:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776625.979805:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1775776625.979810:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000008:00000001:3.0:1775776625.979811:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776625.979815:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776625.979819:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775776625.979822:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776625.979824:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776625.979825:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775776625.979827:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776625.979828:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776625.979831:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776625.979833:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775776625.979835:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111ca00. 02000000:00000010:3.0:1775776625.979838:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34c00. 00000100:00000001:3.0:1775776625.979842:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775776625.979845:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00100000:3.0:1775776625.979850:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710428672/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.0:1775776625.979852:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000100:00000001:3.0:1775776625.979862:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776625.979866:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775776625.979866:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000040:3.0:1775776625.979873:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710428672/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1775776625.979873:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775776625.979888:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776625.979889:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775776625.979892:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111ca00. 00000100:00000040:3.0:1775776625.979894:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776625.979899:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776625.979902:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776625.979905:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776625.979910:0:141703:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13518) 00000400:00000200:0.0:1775776625.979915:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776625.979923:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776625.979930:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0908 00000400:00000010:0.0:1775776625.979933:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0908. 00000100:00000001:0.0:1775776625.979938:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776625.979942:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71500 x1862022710428416/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776650 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.979951:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776625.979954:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71500 x1862022710428416/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776650 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.979961:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776625.979963:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776625.979968:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776625.979978:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776625.979981:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308184838 00000100:00000010:0.0:1775776625.979986:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c00. 00000100:00000001:0.0:1775776625.979994:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776625.980000:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308184838 00000100:00000010:0.0:1775776625.980003:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3840. 00000100:00000001:0.0:1775776625.980007:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776625.980011:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776625.980013:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776625.980015:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776625.980019:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776625/real 1775776625] req@ffff8f6202c71500 x1862022710428416/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776625.980029:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776625.980030:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776625.980032:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d68. 00000400:00000200:0.0:1775776625.980035:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776625.980037:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776625.980039:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776625.980043:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71500 x1862022710428416/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776625.980051:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71500 x1862022710428416/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.980057:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776625.980059:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776625.980061:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776625.980062:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776625.980075:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71500 x1862022710428416/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.980082:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776625.980086:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71500 x1862022710428416/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.980093:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776625.980095:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776625.980097:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776625.980098:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776625.980100:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776625.980102:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776625.980105:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776625.980110:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776625.980112:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776625.980116:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71500 x1862022710428416/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776625.980124:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710428416:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776625.980130:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776625.980132:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776625.980167:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71500 x1862022710428416/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.980174:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776625.980177:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776625.980179:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776625.980183:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776625.980184:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776625.980186:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776625.980188:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776625.980190:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36400. 02000000:00000001:0.0:1775776625.980194:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776625.980196:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776625.980198:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71500. 00000100:00000001:0.0:1775776625.980202:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776625.980203:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776625.980205:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776625.980212:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776625.980214:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776625.980216:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776625.980218:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776625.980219:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776625.980222:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71c00 x1862022710428544/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.980229:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776625.980230:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776625.980232:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776625.980251:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776625.980253:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776625.980257:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710428544:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776625.980262:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776625.980264:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776625.980265:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776625.980268:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776625.980271:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776625.980274:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776625.980277:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776625.980280:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d68. 00000400:00000010:0.0:1775776625.980284:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776625.980287:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710428544, portal 10 00000100:00000001:0.0:1775776625.980289:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776625.980290:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849011200 : -123823860540416 : ffff8f6202c71c00) 00000100:00000040:0.0:1775776625.980296:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71c00 x1862022710428544/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776650 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.980302:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776625.980305:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776625.980307:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0908. 00000100:00000200:0.0:1775776625.980309:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710428544, offset 0 00000400:00000200:0.0:1775776625.980313:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776625.980317:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776625.980321:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776625.980326:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776625.980329:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776625.980331:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776625.980334:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776625.980337:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453900:453900:256:4294967295] 192.168.201.115@tcp LPNI seq info [453825:453825:8:4294967295] 00000400:00000200:0.0:1775776625.980341:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776625.980348:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776625.980352:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776625.980356:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111ca00. 00000800:00000200:0.0:1775776625.980360:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776625.980363:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775776625.980368:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776625.980370:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775776625.980373:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776625.980374:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111ca00. 00000400:00000200:0.0:1775776625.980377:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776625.980380:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776625.980383:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0908 00000400:00000010:0.0:1775776625.980384:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0908. 00000100:00000001:0.0:1775776625.980386:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776625.980389:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71c00 x1862022710428544/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776650 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.980396:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776625.980398:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71c00 x1862022710428544/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776650 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.980404:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776625.980405:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776625.980408:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776625.980412:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776625.980415:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776625.980417:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308181004 00000100:00000001:0.0:1775776625.980420:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308184838 00000100:00000001:0.0:1775776625.980421:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308181004 00000100:00000001:0.0:1775776625.980423:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308184838 00000100:00000001:0.0:1775776625.980425:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776625.980427:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776625.980428:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776625.980432:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776625/real 1775776625] req@ffff8f6202c71c00 x1862022710428544/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776625.980438:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776625.980439:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776625.980441:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d68. 00000400:00000200:0.0:1775776625.980443:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776625.980445:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776625.980446:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776625.980449:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71c00 x1862022710428544/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776625.980456:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71c00 x1862022710428544/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.980461:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776625.980462:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776625.980464:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776625.980465:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776625.980468:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71c00 x1862022710428544/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.980473:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776625.980476:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71c00 x1862022710428544/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.980482:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776625.980484:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776625.980486:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776625.980487:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776625.980488:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776625.980493:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776625.980495:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776625.980498:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776625.980500:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776625.980503:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71c00 x1862022710428544/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776625.980510:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710428544:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776625.980513:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776625.980516:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70380 x1862022710428672/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.980526:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776625.980528:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776625.980530:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776625.980531:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776625.980533:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776625.980535:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710428672:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776625.980538:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776625.980540:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776625.980541:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776625.980542:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776625.980545:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776625.980547:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776625.980550:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776625.980553:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d68. 00000400:00000010:0.0:1775776625.980556:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776625.980558:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710428672, portal 10 00000100:00000001:0.0:1775776625.980560:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776625.980561:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004928 : -123823860546688 : ffff8f6202c70380) 00000100:00000040:0.0:1775776625.980567:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70380 x1862022710428672/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776650 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.980573:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776625.980575:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776625.980577:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0908. 00000100:00000200:0.0:1775776625.980579:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710428672, offset 0 00000400:00000200:0.0:1775776625.980582:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776625.980585:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776625.980589:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776625.980592:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776625.980594:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776625.980602:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776625.980605:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776625.980608:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453901:453901:256:4294967295] 192.168.201.115@tcp LPNI seq info [453826:453826:8:4294967295] 00000400:00000200:0.0:1775776625.980612:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776625.980618:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776625.980622:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776625.980625:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111ca00. 00000800:00000200:0.0:1775776625.980628:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776625.980631:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1775776625.980634:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776625.980637:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1775776625.980640:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776625.980642:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111ca00. 00000400:00000200:0.0:1775776625.980644:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776625.980647:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776625.980650:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0908 00000400:00000010:0.0:1775776625.980651:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0908. 00000100:00000001:0.0:1775776625.980654:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776625.980657:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70380 x1862022710428672/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776650 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.980663:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776625.980666:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70380 x1862022710428672/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776650 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.980672:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776625.980673:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776625.980676:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776625.980678:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776625.980681:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776625.980683:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308181004 00000100:00000001:0.0:1775776625.980685:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308184838 00000100:00000001:0.0:1775776625.980687:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308181004 00000100:00000001:0.0:1775776625.980689:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308184838 00000100:00000001:0.0:1775776625.980691:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776625.980693:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776625.980695:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776625.980698:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776625/real 1775776625] req@ffff8f6202c70380 x1862022710428672/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776625.980705:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776625.980706:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776625.980707:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d68. 00000400:00000200:0.0:1775776625.980710:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776625.980711:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776625.980713:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776625.980716:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70380 x1862022710428672/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776625.980723:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70380 x1862022710428672/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.980729:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776625.980730:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776625.980732:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776625.980733:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776625.980736:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70380 x1862022710428672/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.980742:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776625.980745:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70380 x1862022710428672/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.980751:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776625.980753:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776625.980755:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776625.980756:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776625.980757:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776625.980759:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776625.980761:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776625.980764:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776625.980766:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776625.980769:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70380 x1862022710428672/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776625.980776:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710428672:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776625.980780:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776625.980781:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776625.980784:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71c00 x1862022710428544/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.980790:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776625.980791:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776625.980793:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776625.980795:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776625.980797:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776625.980798:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776625.980800:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776625.980801:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36000. 02000000:00000001:0.0:1775776625.980804:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776625.980805:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776625.980806:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71c00. 00000100:00000001:0.0:1775776625.980808:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776625.980810:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776625.980811:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776625.980814:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70380 x1862022710428672/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776650 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776625.980820:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776625.980821:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776625.980822:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776625.980824:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776625.980825:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776625.980826:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776625.980828:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776625.980830:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34c00. 02000000:00000001:0.0:1775776625.980832:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776625.980833:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776625.980834:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70380. 00000100:00000001:0.0:1775776625.980836:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776625.980837:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776625.980838:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776625.980842:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776625.980843:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776625.980845:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776625.980846:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776626.107285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776626.107289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776626.107294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776626.107294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776626.107297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776626.107298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776626.107360:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776626.107364:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776626.107367:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776627.003267:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776627.003275:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776627.003277:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776627.003282:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776627.003285:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776627.003288:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776627.003291:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776627.003293:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776627.003296:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776627.131311:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776627.131322:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776627.131325:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776627.131329:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776627.131334:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776627.131337:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776627.131339:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776627.131342:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776627.131347:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776627.131349:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776627.131353:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776627.131354:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776627.131412:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776627.131415:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776627.131419:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776628.027303:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776628.027308:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776628.027309:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776628.027314:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776628.027322:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776628.027325:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776628.027327:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776628.027329:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776628.027331:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776628.155232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776628.155237:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776628.155243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776628.155244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776628.155247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776628.155248:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776628.155309:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776628.155313:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776628.155317:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776629.051301:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776629.051306:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776629.051308:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776629.051317:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776629.051320:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776629.051323:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776629.051326:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776629.051328:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776629.051330:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776629.179285:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776629.179291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776629.179296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776629.179297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776629.179301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776629.179302:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776629.179363:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776629.179365:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776629.179368:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776629.883265:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308184838, at 4308184896 00000100:00000010:1.0:1775776629.883294:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3840. 00000100:00000040:0.0:1775776629.883362:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308184838, at 4308184896 00000100:00000010:0.0:1775776629.883369:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c00. 00000100:00000001:0.0:1775776630.075315:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776630.075321:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776630.075323:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776630.075330:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776630.075333:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776630.075337:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776630.075339:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776630.075346:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776630.075349:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776630.203266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776630.203272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776630.203277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776630.203278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776630.203282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776630.203283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776630.203370:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776630.203382:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776630.203389:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776631.099292:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776631.099302:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776631.099304:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776631.099309:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776631.099314:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776631.099321:0:141703:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13513 00000400:00000200:3.0:1775776631.099330:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776631.099337:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776631.099339:0:141703:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776631.099342:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776631.099345:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776631.099347:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776631.099349:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776631.099353:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776631.099355:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776631.099358:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776631.099360:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776631.099363:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776631.099365:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776631.099367:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776631.099369:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776631.099370:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776631.099372:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776631.099374:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776631.099377:0:141703:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776631.099379:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776631.099383:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776631.099386:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776631.099389:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776631.099391:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776631.099394:0:141703:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776631.099402:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775776631.099405:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776631.099409:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776631.099412:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776631.099415:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776631.099418:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776631.099422:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776631.099430:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34400. 00000100:00000001:3.0:1775776631.099437:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776631.099446:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710428800/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776631.099455:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776631.099460:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776631.099465:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710428800/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776631.099487:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776631.099491:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776631.099494:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776631.099496:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776631.099497:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776631.099500:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776631.099511:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13513 00000400:00000200:3.0:1775776631.099516:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776631.099519:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776631.099521:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776631.099524:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776631.099526:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776631.099528:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776631.099530:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776631.099532:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776631.099533:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776631.099535:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776631.099537:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776631.099539:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776631.099542:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776631.099544:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776631.099546:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776631.099547:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776631.099549:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776631.099551:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776631.099554:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776631.099556:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776631.099557:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776631.099563:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776631.099565:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776631.099567:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776631.099571:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776631.099573:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776631.099575:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776631.099578:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000020:00000040:3.0:1775776631.099581:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776631.099583:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776631.099584:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776631.099586:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776631.099587:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776631.099589:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776631.099592:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34800. 00000100:00000001:3.0:1775776631.099595:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776631.099600:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710428928/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776631.099607:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776631.099608:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776631.099612:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710428928/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776631.099619:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776631.099622:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776631.099625:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1775776631.099627:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776631.099628:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776631.099643:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776631.099645:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13513 00000400:00000200:3.0:1775776631.099652:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776631.099654:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776631.099656:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776631.099657:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776631.099659:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776631.099660:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776631.099662:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776631.099663:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776631.099664:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776631.099664:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776631.099666:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776631.099667:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776631.099668:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776631.099670:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776631.099671:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776631.099672:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776631.099673:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776631.099675:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776631.099676:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776631.099677:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776631.099678:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776631.099680:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776631.099681:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776631.099682:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776631.099684:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776631.099685:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776631.099687:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776631.099689:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000020:00000040:3.0:1775776631.099690:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776631.099692:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776631.099693:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776631.099694:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776631.099695:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776631.099696:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776631.099698:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37000. 00000100:00000001:3.0:1775776631.099701:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776631.099704:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710429056/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776631.099709:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776631.099710:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776631.099712:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710429056/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776631.099716:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776631.099718:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776631.099720:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776631.099721:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776631.099723:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776631.099727:0:141703:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13523) 00000100:00000001:0.0:1775776631.099851:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776631.099855:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776631.099857:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776631.099862:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73480 x1862022710428800/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.099867:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776631.099869:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776631.099871:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776631.099873:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776631.099874:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776631.099876:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710428800:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776631.099879:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776631.099882:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776631.099884:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776631.099887:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776631.099891:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776631.099893:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776631.099899:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776631.099903:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d68. 00000400:00000010:0.0:1775776631.099909:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776631.099917:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710428800, portal 25 00000100:00000001:0.0:1775776631.099919:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776631.099921:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849017472 : -123823860534144 : ffff8f6202c73480) 00000100:00000040:0.0:1775776631.099925:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73480 x1862022710428800/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.099933:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776631.099936:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776631.099938:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0908. 00000100:00000200:0.0:1775776631.099941:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710428800, offset 0 00000400:00000200:0.0:1775776631.099947:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776631.099954:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776631.099959:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776631.099964:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776631.099968:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776631.099971:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776631.099974:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776631.099978:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453902:453902:256:4294967295] 192.168.201.115@tcp LPNI seq info [453827:453827:8:4294967295] 00000400:00000200:0.0:1775776631.099982:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776631.099991:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776631.099997:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776631.100002:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111ca00. 00000800:00000200:0.0:1775776631.100009:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776631.100012:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775776631.100017:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776631.100020:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775776631.100024:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776631.100026:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111ca00. 00000400:00000200:0.0:1775776631.100031:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776631.100035:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776631.100040:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0908 00000400:00000010:0.0:1775776631.100042:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0908. 00000100:00000001:0.0:1775776631.100046:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776631.100050:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73480 x1862022710428800/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100066:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776631.100070:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73480 x1862022710428800/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776656 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100076:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776631.100078:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776631.100083:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776631.100085:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776631.100087:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308189958 00000100:00000010:0.0:1775776631.100091:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c00. 00000100:00000001:0.0:1775776631.100096:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776631.100100:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308189958 00000100:00000010:0.0:1775776631.100102:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775776631.100104:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776631.100106:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776631.100108:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776631.100109:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776631.100112:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776631/real 1775776631] req@ffff8f6202c73480 x1862022710428800/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776631.100118:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776631.100119:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776631.100121:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d68. 00000400:00000200:0.0:1775776631.100123:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776631.100124:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776631.100125:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776631.100127:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73480 x1862022710428800/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776631.100132:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73480 x1862022710428800/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100168:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776631.100170:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776631.100173:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776631.100174:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776631.100178:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73480 x1862022710428800/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100185:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776631.100194:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73480 x1862022710428800/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100202:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776631.100204:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776631.100206:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776631.100207:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776631.100210:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776631.100211:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776631.100214:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776631.100217:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776631.100219:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776631.100221:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73480 x1862022710428800/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776631.100227:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710428800:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776631.100230:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776631.100231:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70000 x1862022710428928/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100235:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776631.100236:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776631.100238:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776631.100239:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776631.100240:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776631.100242:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710428928:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776631.100244:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776631.100245:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776631.100246:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776631.100247:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776631.100250:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776631.100251:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776631.100254:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776631.100256:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d68. 00000400:00000010:0.0:1775776631.100260:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776631.100263:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710428928, portal 10 00000100:00000001:0.0:1775776631.100264:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776631.100266:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004032 : -123823860547584 : ffff8f6202c70000) 00000100:00000040:0.0:1775776631.100271:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70000 x1862022710428928/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100277:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776631.100279:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776631.100281:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0908. 00000100:00000200:0.0:1775776631.100283:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710428928, offset 0 00000400:00000200:0.0:1775776631.100287:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776631.100291:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776631.100294:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776631.100296:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776631.100298:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776631.100300:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776631.100302:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776631.100305:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453903:453903:256:4294967295] 192.168.201.115@tcp LPNI seq info [453828:453828:8:4294967295] 00000400:00000200:0.0:1775776631.100309:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776631.100314:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776631.100318:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776631.100322:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111ca00. 00000800:00000200:0.0:1775776631.100325:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776631.100327:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775776631.100331:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776631.100333:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775776631.100337:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776631.100338:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111ca00. 00000400:00000200:0.0:1775776631.100341:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776631.100344:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776631.100346:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0908 00000400:00000010:0.0:1775776631.100348:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0908. 00000100:00000001:0.0:1775776631.100350:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776631.100353:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70000 x1862022710428928/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100359:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776631.100362:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70000 x1862022710428928/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776656 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100367:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776631.100368:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776631.100371:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776631.100374:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776631.100377:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776631.100378:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308186124 00000100:00000001:0.0:1775776631.100381:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308189958 00000100:00000001:0.0:1775776631.100382:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308186124 00000100:00000001:0.0:1775776631.100384:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308189958 00000100:00000001:0.0:1775776631.100386:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776631.100388:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776631.100390:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776631.100393:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776631/real 1775776631] req@ffff8f6202c70000 x1862022710428928/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776631.100405:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776631.100406:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776631.100407:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d68. 00000400:00000200:0.0:1775776631.100409:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776631.100411:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776631.100412:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776631.100415:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70000 x1862022710428928/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776631.100422:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70000 x1862022710428928/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100427:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776631.100428:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776631.100430:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776631.100431:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776631.100434:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70000 x1862022710428928/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100440:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776631.100443:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70000 x1862022710428928/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100449:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776631.100451:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776631.100452:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776631.100453:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776631.100454:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776631.100458:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776631.100461:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776631.100463:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776631.100465:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776631.100469:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70000 x1862022710428928/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776631.100476:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710428928:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776631.100479:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776631.100482:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70e00 x1862022710429056/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100487:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776631.100489:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776631.100490:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776631.100492:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776631.100493:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776631.100496:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710429056:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776631.100498:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776631.100500:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776631.100502:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776631.100503:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776631.100506:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775776631.100508:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776631.100512:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776631.100514:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d68. 00000400:00000010:0.0:1775776631.100516:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776631.100519:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710429056, portal 10 00000100:00000001:0.0:1775776631.100521:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776631.100522:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849007616 : -123823860544000 : ffff8f6202c70e00) 00000100:00000040:0.0:1775776631.100527:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70e00 x1862022710429056/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100533:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776631.100535:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776631.100538:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0908. 00000100:00000200:0.0:1775776631.100540:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710429056, offset 0 00000400:00000200:0.0:1775776631.100544:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776631.100548:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776631.100551:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776631.100553:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776631.100556:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776631.100558:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776631.100560:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776631.100563:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453904:453904:256:4294967295] 192.168.201.115@tcp LPNI seq info [453829:453829:8:4294967295] 00000400:00000200:0.0:1775776631.100567:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776631.100573:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776631.100577:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776631.100579:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111ca00. 00000800:00000200:0.0:1775776631.100583:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776631.100585:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1775776631.100589:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776631.100591:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1775776631.100595:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776631.100597:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111ca00. 00000400:00000200:0.0:1775776631.100599:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776631.100603:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776631.100606:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0908 00000400:00000010:0.0:1775776631.100607:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0908. 00000100:00000001:0.0:1775776631.100610:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776631.100613:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70e00 x1862022710429056/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100620:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776631.100623:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70e00 x1862022710429056/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776656 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100629:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776631.100631:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776631.100634:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776631.100637:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776631.100640:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776631.100643:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308186124 00000100:00000001:0.0:1775776631.100645:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308189958 00000100:00000001:0.0:1775776631.100648:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308186124 00000100:00000001:0.0:1775776631.100649:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308189958 00000100:00000001:0.0:1775776631.100651:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776631.100653:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776631.100655:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776631.100658:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776631/real 1775776631] req@ffff8f6202c70e00 x1862022710429056/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776631.100663:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776631.100664:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776631.100665:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d68. 00000400:00000200:0.0:1775776631.100667:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776631.100668:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776631.100669:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776631.100671:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70e00 x1862022710429056/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776631.100676:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70e00 x1862022710429056/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100679:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776631.100680:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776631.100681:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776631.100682:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776631.100683:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70e00 x1862022710429056/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100687:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776631.100689:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70e00 x1862022710429056/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100692:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776631.100694:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776631.100695:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776631.100696:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776631.100697:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776631.100698:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776631.100699:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776631.100701:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776631.100702:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776631.100704:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70e00 x1862022710429056/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776631.100708:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710429056:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776631.100710:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776631.100712:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776631.100713:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73480 x1862022710428800/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100717:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776631.100719:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776631.100721:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776631.100722:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776631.100724:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776631.100724:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776631.100726:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776631.100727:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34400. 02000000:00000001:0.0:1775776631.100730:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776631.100731:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776631.100733:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73480. 00000100:00000001:0.0:1775776631.100734:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776631.100735:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776631.100736:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776631.100738:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70000 x1862022710428928/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100745:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776631.100746:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776631.100747:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776631.100749:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776631.100750:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776631.100751:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776631.100752:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776631.100754:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34800. 02000000:00000001:0.0:1775776631.100755:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776631.100756:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776631.100757:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70000. 00000100:00000001:0.0:1775776631.100759:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776631.100760:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776631.100761:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776631.100764:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70e00 x1862022710429056/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776656 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776631.100769:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776631.100770:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776631.100771:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775776631.100773:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776631.100774:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776631.100775:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776631.100777:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776631.100778:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37000. 02000000:00000001:0.0:1775776631.100780:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776631.100781:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776631.100783:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70e00. 00000100:00000001:0.0:1775776631.100785:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776631.100786:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776631.100788:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776631.100794:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776631.100796:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776631.100798:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776631.100800:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776631.100860:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776631.100861:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776631.100863:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776631.100865:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776631.100866:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776631.100868:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776631.100869:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776631.100871:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776631.100872:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776631.227272:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776631.227276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776631.227280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776631.227280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776631.227283:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776631.227284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776631.227367:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776631.227370:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776631.227372:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776632.123318:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776632.123328:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776632.123330:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776632.123337:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776632.123342:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776632.123347:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776632.123351:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776632.123354:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776632.123356:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776632.251232:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776632.251240:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776632.251247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776632.251251:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776632.251253:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776632.251257:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776632.251260:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776632.251263:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776632.251265:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776632.251267:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776632.251271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776632.251272:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776632.251274:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776632.251275:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776632.251277:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776632.251278:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776632.251280:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776632.251282:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776632.251342:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776632.251347:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776632.251351:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776633.147269:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776633.147276:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776633.147278:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776633.147284:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776633.147287:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776633.147292:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776633.147294:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776633.147296:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776633.147298:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776633.275289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776633.275296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776633.275303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776633.275304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776633.275307:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776633.275308:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776633.275373:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776633.275378:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776633.275381:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776634.171295:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776634.171303:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776634.171305:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776634.171311:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776634.171319:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776634.171324:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776634.171327:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776634.171331:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776634.171335:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776634.299245:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776634.299250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776634.299253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776634.299254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776634.299257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776634.299257:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776634.299343:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776634.299349:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776634.299353:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776635.003282:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308189958, at 4308190016 00000100:00000040:1.0:1775776635.003282:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308189958, at 4308190016 00000100:00000010:0.0:1775776635.003297:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c00. 00000100:00000010:1.0:1775776635.003302:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775776635.195244:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776635.195250:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776635.195252:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776635.195258:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776635.195261:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776635.195265:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776635.195268:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776635.195270:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776635.195273:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776635.323274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776635.323280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776635.323286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776635.323288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776635.323291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776635.323292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776635.323361:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776635.323365:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776635.323368:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776636.219245:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776636.219252:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1775776636.219254:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776636.219257:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775776636.219258:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776636.219262:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776636.219262:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775776636.219268:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776636.219268:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775776636.219273:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776636.219275:0:141703:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13518 00000100:00000001:0.0:1775776636.219279:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776636.219283:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776636.219286:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1775776636.219287:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776636.219290:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1775776636.219294:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776636.219297:0:141703:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776636.219300:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776636.219303:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776636.219305:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776636.219307:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776636.219310:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776636.219311:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776636.219314:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776636.219316:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776636.219318:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776636.219319:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776636.219321:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776636.219322:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776636.219324:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776636.219325:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776636.219328:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776636.219330:0:141703:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776636.219332:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776636.219335:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776636.219338:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776636.219340:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776636.219341:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776636.219343:0:141703:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776636.219352:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73100. 00000020:00000040:3.0:1775776636.219355:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776636.219358:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776636.219359:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776636.219361:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776636.219364:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776636.219367:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776636.219375:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35400. 00000100:00000001:3.0:1775776636.219382:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776636.219390:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73100 x1862022710429184/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776636.219408:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776636.219414:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776636.219420:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f67e6552] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73100 x1862022710429184/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776636.219446:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776636.219450:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776636.219453:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776636.219454:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776636.219456:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776636.219458:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776636.219460:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13518 00000400:00000200:3.0:1775776636.219464:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776636.219467:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776636.219468:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776636.219471:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776636.219472:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1775776636.219472:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776636.219476:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775776636.219476:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776636.219479:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:0.0:1775776636.219480:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1775776636.219481:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776636.219483:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776636.219485:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000040:0.0:1775776636.219488:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73100 x1862022710429184/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1775776636.219488:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776636.219493:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775776636.219499:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1775776636.219499:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775776636.219502:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776636.219505:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 02000000:00000001:0.0:1775776636.219506:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1775776636.219508:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775776636.219509:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776636.219512:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776636.219512:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776636.219517:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00100000:0.0:1775776636.219518:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710429184:192.168.201.115@tcp:250:kworker.0 00000020:00000040:3.0:1775776636.219522:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000001:0.0:1775776636.219524:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1775776636.219527:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000001:0.0:1775776636.219528:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776636.219531:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776636.219533:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776636.219535:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776636.219539:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776636.219542:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776636.219543:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1775776636.219548:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776636.219550:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1775776636.219552:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d68. 02000000:00000001:3.0:1775776636.219556:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776636.219560:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776636.219568:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710429184, portal 25 00000100:00000001:0.0:1775776636.219569:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776636.219571:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849016576 : -123823860535040 : ffff8f6202c73100) 00000100:00000040:0.0:1775776636.219575:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73100 x1862022710429184/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776636.219580:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776636.219584:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776636.219587:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000008:00100000:3.0:1775776636.219588:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000010:0.0:1775776636.219589:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0908. 00000100:00000200:0.0:1775776636.219594:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710429184, offset 0 00000008:00000001:3.0:1775776636.219594:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776636.219599:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776636.219602:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000010:3.0:1775776636.219606:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70e00. 00000020:00000040:3.0:1775776636.219609:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000400:00000200:0.0:1775776636.219611:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776636.219613:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776636.219616:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776636.219618:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:3.0:1775776636.219620:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776636.219623:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:0.0:1775776636.219625:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1775776636.219625:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776636.219631:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 02000000:00000010:3.0:1775776636.219631:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37400. 00000400:00000200:0.0:1775776636.219636:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776636.219640:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:3.0:1775776636.219644:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776636.219646:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453905:453905:256:4294967295] 192.168.201.115@tcp LPNI seq info [453830:453830:8:4294967295] 00000400:00000200:0.0:1775776636.219652:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00100000:3.0:1775776636.219652:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70e00 x1862022710429312/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775776636.219665:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:3.0:1775776636.219669:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:0.0:1775776636.219673:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776636.219675:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775776636.219679:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111ca00. 00000100:00000040:3.0:1775776636.219680:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be97b27] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70e00 x1862022710429312/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776636.219690:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776636.219692:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.0:1775776636.219693:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00080000:3.0:1775776636.219698:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000200:0.0:1775776636.219699:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000100:00000001:3.0:1775776636.219701:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776636.219704:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:0.1:1775776636.219709:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:3.0:1775776636.219710:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776636.219715:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13518 00000800:00000200:0.1:1775776636.219715:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000400:00000200:3.0:1775776636.219724:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000800:00000100:0.0:1775776636.219725:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776636.219729:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111ca00. 00000400:00000200:3.0:1775776636.219729:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776636.219731:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776636.219734:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776636.219736:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776636.219739:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:0.0:1775776636.219740:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1775776636.219741:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776636.219745:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776636.219747:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000400:00000200:0.0:1775776636.219750:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000020:00000001:3.0:1775776636.219755:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:0.0:1775776636.219756:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0908 00000400:00000010:0.0:1775776636.219759:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0908. 00000020:00000040:3.0:1775776636.219759:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:3.0:1775776636.219764:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000001:0.0:1775776636.219766:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:3.0:1775776636.219767:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000200:0.0:1775776636.219771:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73100 x1862022710429184/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776636.219773:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776636.219776:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776636.219778:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776636.219782:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776636.219788:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00000001:0.0:1775776636.219790:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776636.219795:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73100 x1862022710429184/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776661 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775776636.219799:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776636.219805:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776636.219805:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776636.219809:0:2411:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1775776636.219809:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776636.219815:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776636.219817:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776636.219819:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776636.219820:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776636.219824:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1775776636.219828:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000008:00000001:3.0:1775776636.219829:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776636.219832:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776636.219833:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308195078 00000100:00000010:3.0:1775776636.219836:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70000. 00000100:00000010:0.0:1775776636.219840:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c00. 00000020:00000040:3.0:1775776636.219841:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776636.219846:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776636.219847:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776636.219849:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1775776636.219851:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1775776636.219852:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776636.219855:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776636.219858:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308195078 02000000:00000010:3.0:1775776636.219860:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35800. 00000100:00000010:0.0:1775776636.219862:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3600. 00000100:00000001:3.0:1775776636.219864:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776636.219865:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776636.219870:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1775776636.219871:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70000 x1862022710429440/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776636.219874:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776636.219878:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1775776636.219881:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776636.219883:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776636.219886:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776636/real 1775776636] req@ffff8f6202c73100 x1862022710429184/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776636.219889:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d95bc3d4] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70000 x1862022710429440/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776636.219909:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776636.219911:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776636.219914:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776636.219918:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776636.219921:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000001:0.0:1775776636.219923:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:3.0:1775776636.219925:0:141703:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13528) 00000400:00000001:0.0:1775776636.219926:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776636.219929:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d68. 00000400:00000200:0.0:1775776636.219933:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776636.219935:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776636.219939:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776636.219945:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73100 x1862022710429184/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776636.219954:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73100 x1862022710429184/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776636.219968:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776636.219970:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776636.219973:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776636.219975:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776636.219978:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73100 x1862022710429184/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776636.219983:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776636.219986:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73100 x1862022710429184/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776636.219993:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776636.219996:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776636.219998:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776636.219999:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776636.220001:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776636.220004:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776636.220007:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776636.220011:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776636.220015:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776636.220019:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73100 x1862022710429184/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776636.220027:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710429184:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776636.220032:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776636.220034:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776636.220037:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73100 x1862022710429184/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776636.220044:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776636.220068:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776636.220071:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776636.220075:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776636.220076:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776636.220078:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776636.220081:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776636.220083:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35400. 02000000:00000001:0.0:1775776636.220088:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776636.220090:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776636.220092:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73100. 00000100:00000001:0.0:1775776636.220095:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776636.220096:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776636.220098:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776636.220115:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776636.220117:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776636.220119:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776636.220121:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776636.220122:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776636.220126:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70e00 x1862022710429312/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776636.220154:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776636.220156:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776636.220158:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776636.220160:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776636.220169:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776636.220173:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710429312:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776636.220176:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776636.220178:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776636.220179:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776636.220181:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776636.220185:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776636.220186:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776636.220190:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776636.220192:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d68. 00000400:00000010:0.0:1775776636.220196:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776636.220198:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710429312, portal 10 00000100:00000001:0.0:1775776636.220200:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776636.220202:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849007616 : -123823860544000 : ffff8f6202c70e00) 00000100:00000040:0.0:1775776636.220208:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70e00 x1862022710429312/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776636.220214:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776636.220216:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776636.220217:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0908. 00000100:00000200:0.0:1775776636.220219:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710429312, offset 0 00000400:00000200:0.0:1775776636.220221:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776636.220224:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776636.220227:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776636.220229:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776636.220230:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776636.220232:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776636.220233:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776636.220235:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453906:453906:256:4294967295] 192.168.201.115@tcp LPNI seq info [453831:453831:8:4294967295] 00000400:00000200:0.0:1775776636.220238:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776636.220242:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776636.220244:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776636.220247:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111ca00. 00000800:00000200:0.0:1775776636.220249:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776636.220252:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775776636.220254:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776636.220256:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775776636.220258:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776636.220259:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111ca00. 00000400:00000200:0.0:1775776636.220261:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776636.220263:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776636.220265:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0908 00000400:00000010:0.0:1775776636.220266:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0908. 00000100:00000001:0.0:1775776636.220269:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776636.220272:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70e00 x1862022710429312/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776636.220280:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776636.220282:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70e00 x1862022710429312/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776661 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776636.220289:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776636.220290:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776636.220293:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776636.220296:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776636.220300:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776636.220301:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308191244 00000100:00000001:0.0:1775776636.220305:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308195078 00000100:00000001:0.0:1775776636.220307:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308191244 00000100:00000001:0.0:1775776636.220309:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308195078 00000100:00000001:0.0:1775776636.220312:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776636.220314:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776636.220316:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776636.220319:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776636/real 1775776636] req@ffff8f6202c70e00 x1862022710429312/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776636.220334:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776636.220335:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776636.220337:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d68. 00000400:00000200:0.0:1775776636.220339:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776636.220341:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776636.220343:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776636.220345:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70e00 x1862022710429312/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776636.220354:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70e00 x1862022710429312/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776636.220358:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776636.220358:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776636.220360:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776636.220360:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776636.220363:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70e00 x1862022710429312/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776636.220366:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776636.220369:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70e00 x1862022710429312/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776636.220372:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776636.220373:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776636.220374:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776636.220375:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776636.220376:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776636.220380:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776636.220381:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776636.220383:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776636.220384:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776636.220387:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70e00 x1862022710429312/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776636.220391:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710429312:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776636.220400:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776636.220404:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70000 x1862022710429440/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776636.220408:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776636.220409:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776636.220410:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776636.220412:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776636.220413:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776636.220414:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710429440:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776636.220416:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776636.220417:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776636.220418:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776636.220419:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776636.220421:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775776636.220423:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776636.220426:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776636.220428:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d68. 00000400:00000010:0.0:1775776636.220431:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776636.220434:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710429440, portal 10 00000100:00000001:0.0:1775776636.220435:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776636.220437:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004032 : -123823860547584 : ffff8f6202c70000) 00000100:00000040:0.0:1775776636.220442:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70000 x1862022710429440/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776636.220448:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776636.220450:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776636.220452:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0908. 00000100:00000200:0.0:1775776636.220454:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710429440, offset 0 00000400:00000200:0.0:1775776636.220458:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776636.220461:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776636.220464:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776636.220467:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776636.220470:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776636.220472:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776636.220474:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776636.220477:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453907:453907:256:4294967295] 192.168.201.115@tcp LPNI seq info [453832:453832:8:4294967295] 00000400:00000200:0.0:1775776636.220480:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776636.220484:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776636.220487:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776636.220490:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111ca00. 00000800:00000200:0.0:1775776636.220492:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776636.220494:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1775776636.220496:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776636.220498:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1775776636.220500:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776636.220502:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111ca00. 00000400:00000200:0.0:1775776636.220503:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776636.220506:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776636.220508:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0908 00000400:00000010:0.0:1775776636.220509:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0908. 00000100:00000001:0.0:1775776636.220510:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776636.220512:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70000 x1862022710429440/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776636.220517:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776636.220519:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70000 x1862022710429440/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776661 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776636.220523:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776636.220524:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776636.220526:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776636.220528:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776636.220529:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776636.220531:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308191244 00000100:00000001:0.0:1775776636.220534:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308195078 00000100:00000001:0.0:1775776636.220536:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308191244 00000100:00000001:0.0:1775776636.220538:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308195078 00000100:00000001:0.0:1775776636.220540:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776636.220542:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776636.220544:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776636.220547:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776636/real 1775776636] req@ffff8f6202c70000 x1862022710429440/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776636.220555:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776636.220556:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776636.220558:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d68. 00000400:00000200:0.0:1775776636.220560:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776636.220562:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776636.220564:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776636.220566:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70000 x1862022710429440/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776636.220575:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70000 x1862022710429440/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776636.220581:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776636.220583:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776636.220584:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776636.220586:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776636.220589:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70000 x1862022710429440/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776636.220596:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776636.220599:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70000 x1862022710429440/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776636.220603:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776636.220605:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776636.220606:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776636.220607:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776636.220608:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776636.220610:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776636.220611:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776636.220613:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776636.220615:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776636.220619:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70000 x1862022710429440/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776636.220626:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710429440:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776636.220630:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776636.220631:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776636.220634:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70e00 x1862022710429312/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776636.220640:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776636.220642:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776636.220643:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776636.220646:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776636.220647:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776636.220649:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776636.220651:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776636.220652:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37400. 02000000:00000001:0.0:1775776636.220655:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776636.220656:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776636.220658:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70e00. 00000100:00000001:0.0:1775776636.220661:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776636.220662:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776636.220664:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776636.220667:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70000 x1862022710429440/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776661 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776636.220680:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776636.220681:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776636.220683:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775776636.220685:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776636.220686:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776636.220687:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776636.220689:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776636.220691:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35800. 02000000:00000001:0.0:1775776636.220692:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776636.220694:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776636.220695:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70000. 00000100:00000001:0.0:1775776636.220697:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776636.220698:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776636.220700:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776636.220703:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776636.220704:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776636.220706:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776636.220708:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776636.347282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776636.347292:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776636.347294:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776636.347299:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776636.347304:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776636.347307:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776636.347309:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776636.347312:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776636.347316:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776636.347318:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776636.347322:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776636.347323:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776636.347427:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776636.347431:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776636.347434:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776637.243250:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776637.243257:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776637.243259:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776637.243265:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776637.243268:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776637.243272:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776637.243275:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776637.243277:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776637.243280:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776637.371257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776637.371264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776637.371270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776637.371271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776637.371275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776637.371277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776637.371351:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776637.371354:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776637.371358:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776638.267310:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776638.267317:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776638.267318:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776638.267324:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776638.267328:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776638.267332:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776638.267335:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776638.267337:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776638.267340:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776638.395262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776638.395268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776638.395273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776638.395274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776638.395278:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776638.395279:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776638.395340:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776638.395343:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776638.395346:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776639.291241:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776639.291246:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776639.291248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776639.291252:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776639.291254:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776639.291257:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776639.291259:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776639.291261:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776639.291263:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776639.419266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776639.419272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776639.419277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776639.419278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776639.419282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776639.419283:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776639.419344:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776639.419348:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776639.419351:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776640.123225:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308195078, at 4308195136 00000100:00000010:1.0:1775776640.123239:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3600. 00000100:00000040:0.0:1775776640.124053:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308195078, at 4308195136 00000100:00000010:0.0:1775776640.124061:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c00. 00000100:00000001:0.0:1775776640.315223:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776640.315228:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776640.315229:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776640.315233:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776640.315235:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776640.315238:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776640.315240:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776640.315242:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776640.315243:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776640.443227:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776640.443238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776640.443244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776640.443246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776640.443250:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776640.443251:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776640.443302:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776640.443305:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776640.443309:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776641.339275:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776641.339288:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776641.339289:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776641.339297:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:3.0:1775776641.339297:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776641.339301:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776641.339302:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776641.339304:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776641.339305:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775776641.339308:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1775776641.339309:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776641.339312:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776641.339312:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775776641.339316:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776641.339321:0:141703:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13523 00000400:00000200:3.0:1775776641.339331:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776641.339338:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776641.339341:0:141703:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776641.339345:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776641.339348:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776641.339351:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776641.339354:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776641.339358:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776641.339361:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776641.339364:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776641.339367:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776641.339370:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776641.339373:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776641.339376:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776641.339378:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776641.339381:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776641.339383:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776641.339386:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776641.339389:0:141703:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776641.339393:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776641.339396:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776641.339399:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776641.339402:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776641.339405:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776641.339407:0:141703:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776641.339415:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c73480. 00000020:00000040:3.0:1775776641.339419:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776641.339422:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776641.339424:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776641.339426:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776641.339429:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776641.339433:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776641.339440:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37800. 00000100:00000001:3.0:1775776641.339447:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776641.339456:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c73480 x1862022710429568/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776641.339466:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776641.339470:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776641.339474:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007e51d1c8] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c73480 x1862022710429568/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776641.339499:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776641.339502:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776641.339505:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776641.339507:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776641.339508:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776641.339511:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776641.339514:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13523 00000400:00000200:3.0:1775776641.339518:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776641.339521:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00000001:0.0:1775776641.339523:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1775776641.339524:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775776641.339528:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1775776641.339528:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776641.339532:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1775776641.339532:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776641.339535:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776641.339538:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:0.0:1775776641.339541:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c73480 x1862022710429568/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775776641.339541:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776641.339542:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776641.339544:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776641.339547:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776641.339551:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776641.339556:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775776641.339558:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1775776641.339560:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776641.339561:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776641.339563:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776641.339565:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 02000000:00000001:0.0:1775776641.339566:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1775776641.339567:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 02000000:00000001:0.0:1775776641.339570:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1775776641.339571:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 02000000:00000001:0.0:1775776641.339576:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776641.339577:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00100000:0.0:1775776641.339582:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710429568:192.168.201.115@tcp:250:kworker.0 00000100:00000001:3.0:1775776641.339582:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776641.339586:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1775776641.339590:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776641.339593:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1775776641.339593:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776641.339596:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776641.339597:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776641.339599:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775776641.339600:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000010:0.0:1775776641.339605:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4800. 00000008:00100000:3.0:1775776641.339606:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1775776641.339607:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1775776641.339612:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776641.339616:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776641.339616:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776641.339619:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d68. 00000100:00000010:3.0:1775776641.339621:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70380. 00000020:00000040:3.0:1775776641.339624:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000400:00000010:0.0:1775776641.339626:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 02000000:00000001:3.0:1775776641.339626:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000200:0.0:1775776641.339632:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710429568, portal 25 02000000:00000001:3.0:1775776641.339633:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776641.339636:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1775776641.339637:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1775776641.339640:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849017472 : -123823860534144 : ffff8f6202c73480) 02000000:00000001:3.0:1775776641.339640:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776641.339643:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776641.339648:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35000. 00000100:00000001:3.0:1775776641.339651:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776641.339659:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70380 x1862022710429696/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1775776641.339660:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c73480 x1862022710429568/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776666 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776641.339682:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776641.339684:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775776641.339685:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776641.339690:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776641.339693:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000506baf00] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70380 x1862022710429696/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1775776641.339694:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0908. 00000100:00000200:0.0:1775776641.339699:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710429568, offset 0 00000100:00000001:3.0:1775776641.339708:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776641.339711:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776641.339715:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:0.0:1775776641.339718:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776641.339718:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776641.339721:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776641.339724:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776641.339728:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13523 00000400:00000200:0.0:1775776641.339731:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776641.339735:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776641.339739:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:3.0:1775776641.339740:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776641.339744:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1775776641.339747:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1775776641.339749:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776641.339752:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775776641.339754:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000040:3.0:1775776641.339757:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:0.0:1775776641.339760:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1775776641.339761:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:0.0:1775776641.339765:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000020:00000001:3.0:1775776641.339765:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776641.339768:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000400:00000200:0.0:1775776641.339770:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453908:453908:256:4294967295] 192.168.201.115@tcp LPNI seq info [453833:453833:8:4294967295] 00000020:00000001:3.0:1775776641.339771:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776641.339774:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000400:00000200:0.0:1775776641.339777:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000001:3.0:1775776641.339778:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776641.339781:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776641.339785:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:0.0:1775776641.339790:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:3.0:1775776641.339790:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776641.339795:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776641.339798:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000800:00000200:0.0:1775776641.339799:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000020:00000040:3.0:1775776641.339802:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000800:00000010:0.0:1775776641.339805:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111ca00. 00000100:00080000:3.0:1775776641.339807:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776641.339810:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776641.339812:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:0.0:1775776641.339813:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:3.0:1775776641.339815:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:0.0:1775776641.339817:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 02000000:00000001:3.0:1775776641.339820:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776641.339824:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000800:00000200:0.1:1775776641.339824:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000008:00100000:3.0:1775776641.339827:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000200:0.1:1775776641.339828:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000008:00000001:3.0:1775776641.339832:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1775776641.339835:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776641.339836:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775776641.339838:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111ca00. 00000100:00000010:3.0:1775776641.339841:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71c00. 00000400:00000200:0.0:1775776641.339843:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:3.0:1775776641.339844:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776641.339848:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1775776641.339849:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 02000000:00000001:3.0:1775776641.339851:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776641.339854:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0908 00000100:00000001:3.0:1775776641.339854:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:0.0:1775776641.339857:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0908. 02000000:00000001:3.0:1775776641.339857:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776641.339859:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776641.339862:0:2411:0:(events.c:34:request_out_callback()) Process entered 02000000:00000010:3.0:1775776641.339864:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc37c00. 00000100:00000200:0.0:1775776641.339866:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c73480 x1862022710429568/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776666 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776641.339870:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776641.339876:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776641.339880:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c73480 x1862022710429568/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776666 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775776641.339885:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71c00 x1862022710429824/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776641.339886:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776641.339891:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776641.339897:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776641.339897:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776641.339902:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776641.339905:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776641.339909:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308200198 00000100:00000040:3.0:1775776641.339910:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d91503f] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71c00 x1862022710429824/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1775776641.339917:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c00. 00000100:00000001:3.0:1775776641.339923:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776641.339926:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000040:3.0:1775776641.339926:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776641.339931:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1775776641.339933:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308200198 00000100:00000040:3.0:1775776641.339934:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000010:0.0:1775776641.339936:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3ba0. 00000100:00000001:0.0:1775776641.339941:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000040:3.0:1775776641.339941:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1775776641.339947:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776641.339949:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776641.339949:0:141703:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13534) 00000100:00000001:0.0:1775776641.339953:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776641.339957:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776641/real 1775776641] req@ffff8f6202c73480 x1862022710429568/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776641.339964:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776641.339976:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776641.339979:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d68. 00000400:00000200:0.0:1775776641.339982:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776641.339983:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776641.339986:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776641.339989:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c73480 x1862022710429568/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776641.339997:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c73480 x1862022710429568/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776641.340003:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776641.340004:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776641.340006:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776641.340008:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776641.340012:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c73480 x1862022710429568/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776641.340016:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776641.340019:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c73480 x1862022710429568/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776641.340023:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776641.340025:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776641.340026:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776641.340026:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776641.340028:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776641.340029:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776641.340031:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776641.340034:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776641.340035:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776641.340037:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c73480 x1862022710429568/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776641.340042:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c73480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710429568:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776641.340045:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776641.340046:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776641.340047:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c73480 x1862022710429568/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776641.340051:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776641.340053:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776641.340054:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775776641.340056:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776641.340057:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776641.340058:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776641.340059:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776641.340061:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37800. 02000000:00000001:0.0:1775776641.340063:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776641.340065:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776641.340065:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c73480. 00000100:00000001:0.0:1775776641.340068:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776641.340068:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776641.340070:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776641.340075:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776641.340076:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776641.340077:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776641.340078:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776641.340079:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776641.340081:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70380 x1862022710429696/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776641.340085:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776641.340086:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776641.340088:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776641.340089:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776641.340091:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776641.340092:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710429696:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776641.340094:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776641.340096:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776641.340096:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776641.340097:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776641.340099:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775776641.340100:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776641.340103:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776641.340105:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d68. 00000400:00000010:0.0:1775776641.340107:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776641.340109:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710429696, portal 10 00000100:00000001:0.0:1775776641.340110:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776641.340111:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849004928 : -123823860546688 : ffff8f6202c70380) 00000100:00000040:0.0:1775776641.340115:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70380 x1862022710429696/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776666 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776641.340119:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776641.340120:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776641.340122:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0908. 00000100:00000200:0.0:1775776641.340123:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710429696, offset 0 00000400:00000200:0.0:1775776641.340125:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776641.340128:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776641.340130:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776641.340131:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776641.340151:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776641.340153:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776641.340155:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776641.340157:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453909:453909:256:4294967295] 192.168.201.115@tcp LPNI seq info [453834:453834:8:4294967295] 00000400:00000200:0.0:1775776641.340160:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776641.340163:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776641.340167:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776641.340169:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111ca00. 00000800:00000200:0.0:1775776641.340172:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776641.340173:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775776641.340176:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776641.340177:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1775776641.340179:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776641.340181:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111ca00. 00000400:00000200:0.0:1775776641.340182:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776641.340184:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776641.340186:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0908 00000400:00000010:0.0:1775776641.340187:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0908. 00000100:00000001:0.0:1775776641.340189:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776641.340191:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70380 x1862022710429696/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776666 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776641.340195:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776641.340197:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70380 x1862022710429696/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776666 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776641.340201:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776641.340202:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776641.340203:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776641.340206:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776641.340208:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776641.340209:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308196364 00000100:00000001:0.0:1775776641.340211:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308200198 00000100:00000001:0.0:1775776641.340212:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308196364 00000100:00000001:0.0:1775776641.340213:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308200198 00000100:00000001:0.0:1775776641.340214:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776641.340215:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776641.340216:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776641.340218:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776641/real 1775776641] req@ffff8f6202c70380 x1862022710429696/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776641.340223:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776641.340224:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776641.340225:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d68. 00000400:00000200:0.0:1775776641.340226:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776641.340227:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776641.340228:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776641.340230:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70380 x1862022710429696/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776641.340234:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70380 x1862022710429696/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776641.340238:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776641.340239:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776641.340240:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776641.340240:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776641.340242:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70380 x1862022710429696/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776641.340246:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776641.340248:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70380 x1862022710429696/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776641.340251:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776641.340252:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776641.340253:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776641.340254:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776641.340255:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776641.340258:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776641.340259:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776641.340261:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776641.340262:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776641.340264:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70380 x1862022710429696/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776641.340269:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710429696:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776641.340271:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776641.340272:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71c00 x1862022710429824/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776641.340276:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776641.340277:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776641.340278:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776641.340279:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776641.340280:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776641.340281:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710429824:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776641.340283:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776641.340283:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776641.340284:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776641.340285:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776641.340287:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776641.340288:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776641.340290:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776641.340291:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d68. 00000400:00000010:0.0:1775776641.340293:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776641.340294:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710429824, portal 10 00000100:00000001:0.0:1775776641.340295:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776641.340296:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849011200 : -123823860540416 : ffff8f6202c71c00) 00000100:00000040:0.0:1775776641.340300:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71c00 x1862022710429824/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776666 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776641.340303:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776641.340305:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776641.340306:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0908. 00000100:00000200:0.0:1775776641.340307:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710429824, offset 0 00000400:00000200:0.0:1775776641.340310:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776641.340312:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776641.340314:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776641.340315:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776641.340317:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776641.340319:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776641.340320:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776641.340322:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453910:453910:256:4294967295] 192.168.201.115@tcp LPNI seq info [453835:453835:8:4294967295] 00000400:00000200:0.0:1775776641.340324:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776641.340328:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776641.340330:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776641.340331:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111ca00. 00000800:00000200:0.0:1775776641.340333:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776641.340335:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1775776641.340337:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776641.340338:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1775776641.340341:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776641.340342:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111ca00. 00000400:00000200:0.0:1775776641.340343:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776641.340345:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776641.340347:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0908 00000400:00000010:0.0:1775776641.340348:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0908. 00000100:00000001:0.0:1775776641.340349:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776641.340351:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71c00 x1862022710429824/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776666 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776641.340355:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776641.340356:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71c00 x1862022710429824/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776666 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776641.340360:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776641.340361:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776641.340362:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776641.340364:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776641.340372:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776641.340374:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308196364 00000100:00000001:0.0:1775776641.340376:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308200198 00000100:00000001:0.0:1775776641.340377:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308196364 00000100:00000001:0.0:1775776641.340378:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308200198 00000100:00000001:0.0:1775776641.340379:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776641.340380:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776641.340381:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776641.340383:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776641/real 1775776641] req@ffff8f6202c71c00 x1862022710429824/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776641.340387:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776641.340388:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776641.340389:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d68. 00000400:00000200:0.0:1775776641.340390:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776641.340391:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776641.340392:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776641.340394:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71c00 x1862022710429824/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776641.340399:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71c00 x1862022710429824/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776641.340402:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776641.340403:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776641.340404:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776641.340404:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776641.340406:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71c00 x1862022710429824/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776641.340409:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776641.340411:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71c00 x1862022710429824/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776641.340416:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776641.340419:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776641.340420:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776641.340421:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776641.340423:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776641.340424:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776641.340426:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776641.340429:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776641.340431:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776641.340434:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71c00 x1862022710429824/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776641.340441:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710429824:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776641.340445:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776641.340446:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776641.340449:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70380 x1862022710429696/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776641.340455:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776641.340457:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776641.340459:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775776641.340461:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776641.340462:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776641.340463:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776641.340465:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776641.340467:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35000. 02000000:00000001:0.0:1775776641.340469:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776641.340471:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776641.340472:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70380. 00000100:00000001:0.0:1775776641.340474:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776641.340475:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776641.340477:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776641.340480:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71c00 x1862022710429824/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776666 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776641.340492:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776641.340493:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776641.340494:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e6000. 02000000:00000001:0.0:1775776641.340496:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776641.340498:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776641.340498:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776641.340500:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776641.340500:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc37c00. 02000000:00000001:0.0:1775776641.340502:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776641.340502:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776641.340503:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71c00. 00000100:00000001:0.0:1775776641.340505:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776641.340505:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776641.340507:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776641.340509:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776641.340510:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776641.340512:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776641.340513:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776641.467303:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776641.467309:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776641.467314:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776641.467328:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776641.467330:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776641.467334:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776641.467337:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776641.467339:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776641.467341:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776641.467343:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776641.467348:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776641.467349:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776641.467350:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776641.467352:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776641.467353:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776641.467354:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776641.467356:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776641.467358:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776641.467456:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776641.467460:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776641.467463:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776642.363369:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776642.363376:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776642.363378:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776642.363383:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776642.363387:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776642.363390:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776642.363392:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776642.363394:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776642.363397:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776642.491242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776642.491249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776642.491255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776642.491256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776642.491260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776642.491261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776642.491313:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776642.491317:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776642.491320:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.0:1775776643.195261:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:3.0:1775776643.195268:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:3.0:1775776643.195273:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1775776643.195682:0:2404:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000001:1.0:1775776643.195684:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1775776643.195700:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1775776643.195704:0:2405:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1775776643.195707:0:2405:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776643.389058:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776643.389065:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776643.389067:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776643.389072:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776643.389076:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776643.389080:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776643.389083:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776643.389085:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776643.389089:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776643.515318:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776643.515325:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776643.515332:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776643.515334:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776643.515337:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776643.515339:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776643.515398:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776643.515403:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776643.515407:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776644.412029:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776644.412039:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776644.412042:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776644.412048:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776644.412053:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776644.412057:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776644.412059:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776644.412062:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776644.412066:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776644.539277:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776644.539284:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776644.539289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776644.539290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776644.539294:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776644.539296:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776644.539432:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776644.539436:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776644.539440:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776645.243275:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308200198, at 4308200256 00000100:00000040:1.0:1775776645.243275:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308200198, at 4308200256 00000100:00000010:0.0:1775776645.243290:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c00. 00000100:00000010:1.0:1775776645.243293:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3ba0. 00000100:00000001:3.0:1775776645.435283:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776645.435292:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776645.435294:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776645.435301:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776645.435305:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776645.435310:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776645.435313:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776645.435317:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776645.435320:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776645.563324:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776645.563333:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776645.563342:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776645.563344:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776645.563349:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776645.563350:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776645.563456:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776645.563459:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776645.563463:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776646.459234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776646.459244:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776646.459246:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776646.459251:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775776646.459254:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776646.459260:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1775776646.459262:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776646.459265:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776646.459266:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775776646.459271:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1775776646.459273:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776646.459275:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776646.459278:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775776646.459280:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776646.459289:0:141703:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13529 00000400:00000200:2.0:1775776646.459301:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776646.459310:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776646.459314:0:141703:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776646.459318:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776646.459322:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776646.459324:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776646.459327:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776646.459331:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776646.459333:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776646.459336:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776646.459339:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776646.459342:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776646.459344:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776646.459347:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776646.459349:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776646.459351:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776646.459353:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776646.459356:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776646.459359:0:141703:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776646.459362:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776646.459366:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776646.459370:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776646.459373:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776646.459376:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776646.459379:0:141703:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776646.459390:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775776646.459394:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776646.459407:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776646.459409:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776646.459413:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776646.459416:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776646.459421:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776646.459431:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775776646.459439:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776646.459450:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710429952/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776646.459460:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776646.459465:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776646.459470:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710429952/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776646.459498:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776646.459502:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776646.459505:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776646.459507:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776646.459508:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776646.459510:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776646.459513:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13529 00000400:00000200:2.0:1775776646.459517:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776646.459520:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776646.459522:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776646.459524:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776646.459526:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776646.459528:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776646.459530:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776646.459531:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776646.459533:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776646.459534:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776646.459537:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776646.459539:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776646.459541:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776646.459543:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776646.459545:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776646.459547:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776646.459548:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776646.459551:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776646.459553:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776646.459555:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776646.459557:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776646.459563:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776646.459565:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776646.459567:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776646.459572:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776646.459574:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776646.459576:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776646.459582:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775776646.459584:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776646.459587:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776646.459588:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776646.459589:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776646.459591:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776646.459592:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776646.459596:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775776646.459599:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776646.459610:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710430080/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776646.459617:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776646.459619:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776646.459622:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710430080/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776646.459635:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776646.459638:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776646.459641:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776646.459642:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776646.459644:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776646.459647:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776646.459649:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13529 00000400:00000200:2.0:1775776646.459653:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776646.459656:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776646.459658:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776646.459661:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776646.459663:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776646.459665:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776646.459667:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776646.459668:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776646.459670:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776646.459671:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776646.459673:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776646.459675:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776646.459677:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776646.459680:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776646.459682:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776646.459683:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776646.459686:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776646.459688:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776646.459690:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776646.459693:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776646.459695:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776646.459697:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776646.459699:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776646.459701:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776646.459703:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776646.459705:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776646.459707:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776646.459710:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775776646.459713:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776646.459715:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776646.459716:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776646.459718:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776646.459719:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776646.459720:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776646.459724:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775776646.459727:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776646.459731:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710430208/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776646.459738:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776646.459739:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776646.459742:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710430208/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776646.459749:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776646.459752:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776646.459757:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776646.459760:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776646.459763:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776646.459769:0:141703:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13539) 00000100:00000001:0.0:1775776646.459826:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776646.459830:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776646.459833:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776646.459839:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344bb80 x1862022710429952/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776646.459846:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776646.459848:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776646.459852:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776646.459855:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776646.459858:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776646.459861:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710429952:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776646.459865:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776646.459870:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776646.459873:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776646.459887:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776646.459893:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775776646.459896:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776646.459902:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776646.459906:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d68. 00000400:00000010:0.0:1775776646.459913:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0990. 00000100:00000200:0.0:1775776646.459922:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710429952, portal 25 00000100:00000001:0.0:1775776646.459925:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776646.459926:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662550400 : -123823047001216 : ffff8f623344bb80) 00000100:00000040:0.0:1775776646.459942:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344bb80 x1862022710429952/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776671 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776646.459954:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776646.459958:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776646.459960:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0908. 00000100:00000200:0.0:1775776646.459963:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710429952, offset 0 00000400:00000200:0.0:1775776646.459970:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776646.459978:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776646.459985:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776646.459991:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776646.459995:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776646.460008:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776646.460011:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776646.460015:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453911:453911:256:4294967295] 192.168.201.115@tcp LPNI seq info [453836:453836:8:4294967295] 00000400:00000200:0.0:1775776646.460020:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776646.460030:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776646.460036:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776646.460041:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111ca00. 00000800:00000200:0.0:1775776646.460055:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776646.460059:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775776646.460076:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776646.460080:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776646.460083:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308205318 00000100:00000010:0.0:1775776646.460088:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c00. 00000100:00000001:0.0:1775776646.460096:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776646.460103:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308205318 00000100:00000010:0.0:1775776646.460106:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3ba0. 00000100:00000001:0.0:1775776646.460109:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776646.460112:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776646.460115:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776646.460118:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776646.460122:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233448700 x1862022710430080/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776646.460151:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776646.460153:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776646.460155:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776646.460157:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776646.460159:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776646.460161:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710430080:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776646.460164:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776646.460166:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776646.460168:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776646.460169:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776646.460173:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776646.460174:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776646.460177:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776646.460180:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f62338886e0. 00000400:00000010:0.0:1775776646.460184:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776646.460186:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710430080, portal 10 00000100:00000001:0.0:1775776646.460188:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776646.460189:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662536960 : -123823047014656 : ffff8f6233448700) 00000100:00000040:0.0:1775776646.460195:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233448700 x1862022710430080/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776671 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776646.460201:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776646.460203:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776646.460205:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0dd0. 00000100:00000200:0.0:1775776646.460207:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710430080, offset 0 00000400:00000200:0.0:1775776646.460211:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776646.460214:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776646.460217:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776646.460220:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776646.460222:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776646.460224:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776646.460226:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776646.460230:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453912:453912:255:4294967295] 192.168.201.115@tcp LPNI seq info [453837:453837:7:4294967295] 00000400:00000200:0.0:1775776646.460234:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776646.460239:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776646.460243:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776646.460246:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111c700. 00000800:00000200:0.0:1775776646.460250:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776646.460253:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775776646.460256:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776646.460260:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776646.460263:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776646.460265:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308201484 00000100:00000001:0.0:1775776646.460267:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308205318 00000100:00000001:0.0:1775776646.460269:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308201484 00000100:00000001:0.0:1775776646.460271:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308205318 00000100:00000001:0.0:1775776646.460273:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776646.460275:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776646.460278:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776646.460281:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449c00 x1862022710430208/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776646.460287:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776646.460289:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776646.460290:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776646.460292:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776646.460294:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776646.460296:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710430208:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776646.460299:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776646.460301:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776646.460302:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776646.460304:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776646.460307:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776646.460309:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776646.460313:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776646.460315:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776646.460318:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0f68. 00000100:00000200:0.0:1775776646.460321:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710430208, portal 10 00000100:00000001:0.0:1775776646.460323:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776646.460324:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662542336 : -123823047009280 : ffff8f6233449c00) 00000100:00000040:0.0:1775776646.460329:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449c00 x1862022710430208/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776671 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776646.460336:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776646.460338:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776646.460340:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0b28. 00000100:00000200:0.0:1775776646.460342:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710430208, offset 0 00000400:00000200:0.0:1775776646.460347:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776646.460350:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776646.460353:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776646.460355:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776646.460357:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776646.460360:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776646.460362:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776646.460364:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453913:453913:254:4294967295] 192.168.201.115@tcp LPNI seq info [453838:453838:6:4294967295] 00000400:00000200:0.0:1775776646.460368:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776646.460373:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776646.460377:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776646.460381:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cf00. 00000800:00000200:0.0:1775776646.460384:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776646.460387:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00000001:0.0:1775776646.460390:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776646.460392:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776646.460395:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776646.460397:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308201484 00000100:00000001:0.0:1775776646.460398:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308205318 00000100:00000001:0.0:1775776646.460400:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308201484 00000100:00000001:0.0:1775776646.460403:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308205318 00000100:00000001:0.0:1775776646.460405:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776646.460406:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776646.460409:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776646.460412:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776646.460419:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776646.460421:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1775776646.460423:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776646.460424:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776646.460425:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776646.460427:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776646.587413:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776646.587427:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776646.587429:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776646.587434:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776646.587440:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776646.587443:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776646.587445:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776646.587448:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776646.587453:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776646.587455:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776646.587459:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776646.587460:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776646.587723:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776646.587726:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776646.587728:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776647.483288:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776647.483296:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776647.483298:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776647.483304:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776647.483308:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776647.483311:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776647.483314:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776647.483316:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776647.483319:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776647.611233:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776647.611238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776647.611244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776647.611246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776647.611249:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776647.611250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776647.611299:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776647.611302:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776647.611305:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776648.507280:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776648.507288:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776648.507290:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776648.507296:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776648.507300:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776648.507303:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776648.507306:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776648.507308:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776648.507310:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776648.635282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776648.635290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776648.635298:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776648.635300:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776648.635304:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776648.635306:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776648.635425:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776648.635429:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776648.635433:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776649.531254:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776649.531260:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776649.531262:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776649.531266:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776649.531269:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776649.531272:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776649.531275:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776649.531276:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776649.531278:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1775776649.531422:0:2400:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.115:988 00000400:00000100:2.0:1775776649.531460:0:2400:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.115@tcp at host 192.168.201.115 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1775776649.531472:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776649.531476:0:2400:0:(router.c:1750:lnet_notify()) 192.168.201.15@tcp notifying 192.168.201.115@tcp: down 00000400:00000200:2.0:1775776649.531481:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776649.531486:0:2400:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:2.0:1775776649.531499:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775776649.531507:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111ca00. 00000400:00000200:2.0:1775776649.531514:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775776649.531519:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775776649.531527:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0908 00000400:00000010:2.0:1775776649.531529:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0908. 00000100:00000001:2.0:1775776649.531535:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776649.531548:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f623344bb80 x1862022710429952/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776671 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776649.531577:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776649.531581:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344bb80 x1862022710429952/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776671 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776649.531588:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776649.531590:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775776649.531600:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775776649.531602:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111c700. 00000400:00000200:2.0:1775776649.531605:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775776649.531608:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775776649.531611:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0dd0 00000400:00000010:2.0:1775776649.531612:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0dd0. 00000100:00000001:2.0:1775776649.531615:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776649.531618:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6233448700 x1862022710430080/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776671 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776649.531626:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776649.531629:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448700 x1862022710430080/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776671 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776649.531636:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776649.531637:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775776649.531642:0:2400:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.115@tcp 00000800:00000010:2.0:1775776649.531644:0:2400:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cf00. 00000400:00000200:2.0:1775776649.531646:0:2400:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775776649.531648:0:2400:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775776649.531651:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0b28 00000400:00000010:2.0:1775776649.531652:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0b28. 00000100:00000001:2.0:1775776649.531654:0:2400:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775776649.531658:0:2400:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8f6233449c00 x1862022710430208/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776671 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776649.531663:0:2400:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775776649.531665:0:2400:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449c00 x1862022710430208/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776671 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776649.531676:0:2400:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776649.531677:0:2400:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776649.532016:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776649.532020:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776649.532022:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776649.532029:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776646/real 1775776649] req@ffff8f623344bb80 x1862022710429952/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776649.532042:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776649.532043:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776649.532047:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d68. 00000400:00000200:0.0:1775776649.532051:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0990 00000400:00000010:0.0:1775776649.532053:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0990. 00000100:00000001:0.0:1775776649.532056:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776649.532060:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344bb80 x1862022710429952/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776649.532070:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344bb80 x1862022710429952/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776649.532076:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776649.532078:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776649.532081:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776649.532083:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776649.532087:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344bb80 x1862022710429952/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776649.532096:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776649.532101:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344bb80 x1862022710429952/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776649.532108:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776649.532111:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776649.532112:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776649.532113:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776649.532116:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776649.532118:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776649.532122:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776649.532129:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776649.532131:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776649.532165:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344bb80 x1862022710429952/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776649.532176:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710429952:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776649.532181:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776649.532185:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776646/real 1775776649] req@ffff8f6233448700 x1862022710430080/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776649.532192:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776649.532194:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776649.532195:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f62338886e0. 00000400:00000200:0.0:1775776649.532198:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776649.532199:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776649.532202:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776649.532205:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233448700 x1862022710430080/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776649.532212:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233448700 x1862022710430080/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776649.532218:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776649.532220:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776649.532221:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776649.532223:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776649.532225:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233448700 x1862022710430080/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776649.532232:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776649.532235:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233448700 x1862022710430080/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776649.532242:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776649.532244:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776649.532245:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776649.532247:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776649.532248:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776649.534910:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776649.534922:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776649.534932:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776649.534935:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776649.534945:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233448700 x1862022710430080/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776649.534957:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710430080:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776649.534964:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776649.534969:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776646/real 1775776649] req@ffff8f6233449c00 x1862022710430208/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776649.534981:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776649.534983:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776649.534987:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776649.534992:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0f68 00000400:00000010:0.0:1775776649.534994:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0f68. 00000100:00000001:0.0:1775776649.534998:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776649.535001:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449c00 x1862022710430208/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776649.535009:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449c00 x1862022710430208/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776649.535016:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776649.535018:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776649.535020:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776649.535022:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776649.535026:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449c00 x1862022710430208/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776649.535031:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776649.535035:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449c00 x1862022710430208/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776649.535041:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776649.535043:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776649.535045:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776649.535046:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776649.535058:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776649.535063:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776649.535066:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776649.535070:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776649.535071:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776649.535075:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449c00 x1862022710430208/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776649.535082:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710430208:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776649.535087:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776649.535090:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776649.535094:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344bb80 x1862022710429952/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776649.535100:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776649.535104:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776649.535108:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:0.0:1775776649.535112:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776649.535115:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776649.535117:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776649.535120:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776649.535122:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4e000. 02000000:00000001:0.0:1775776649.535126:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776649.535129:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776649.535130:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344bb80. 00000100:00000001:0.0:1775776649.535151:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776649.535153:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776649.535154:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776649.535157:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233448700 x1862022710430080/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776649.535162:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776649.535164:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776649.535165:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776649.535167:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776649.535168:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776649.535169:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776649.535171:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776649.535172:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4f000. 02000000:00000001:0.0:1775776649.535174:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776649.535175:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776649.535176:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233448700. 00000100:00000001:0.0:1775776649.535178:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776649.535179:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776649.535180:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776649.535184:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449c00 x1862022710430208/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776671 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776649.535190:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776649.535191:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776649.535192:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776649.535194:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776649.535195:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776649.535196:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776649.535197:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776649.535198:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4ec00. 02000000:00000001:0.0:1775776649.535200:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776649.535201:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776649.535202:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449c00. 00000100:00000001:0.0:1775776649.535204:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776649.535205:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776649.535207:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776649.535214:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776649.535216:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776649.535218:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776649.535220:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776649.659280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776649.659285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776649.659288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776649.659289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776649.659292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776649.659292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776649.659364:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776649.659368:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776649.659372:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776650.363249:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308205318, at 4308205376 00000100:00000040:1.0:1775776650.363249:0:214537:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308205318, at 4308205376 00000100:00000010:0.0:1775776650.363263:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c00. 00000100:00000010:1.0:1775776650.363267:0:214537:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3ba0. 00000100:00000001:3.0:1775776650.555254:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776650.555260:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776650.555262:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776650.555267:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776650.555271:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776650.555274:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776650.555276:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776650.555278:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776650.555280:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776650.683254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776650.683259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776650.683264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776650.683265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776650.683269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776650.683270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776650.683348:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776650.683352:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776650.683355:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776651.579257:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776651.579265:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776651.579267:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776651.579272:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776651.579276:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776651.579280:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776651.579283:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776651.579285:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776651.579288:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776651.579328:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776651.579332:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776651.579335:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776651.579337:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776651.579341:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776651.579348:0:141703:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13534 00000400:00000200:2.0:1775776651.579357:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776651.579363:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776651.579366:0:141703:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776651.579369:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776651.579372:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776651.579375:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776651.579377:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776651.579382:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776651.579384:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776651.579387:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776651.579390:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776651.579392:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776651.579394:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776651.579397:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776651.579399:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776651.579401:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776651.579403:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776651.579405:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776651.579408:0:141703:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776651.579411:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776651.579422:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776651.579428:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776651.579430:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776651.579432:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776651.579433:0:141703:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776651.579440:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775776651.579444:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776651.579447:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776651.579449:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776651.579452:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776651.579455:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776651.579458:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776651.579467:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4cc00. 00000100:00000001:2.0:1775776651.579473:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776651.579481:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710430336/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776651.579489:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776651.579493:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776651.579497:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710430336/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776651.579537:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776651.579541:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776651.579544:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776651.579545:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776651.579547:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776651.579549:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776651.579552:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13534 00000400:00000200:2.0:1775776651.579556:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776651.579558:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776651.579561:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776651.579563:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776651.579566:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776651.579567:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776651.579569:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776651.579571:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776651.579572:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776651.579574:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776651.579575:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776651.579577:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776651.579579:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776651.579582:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776651.579584:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776651.579585:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776651.579587:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776651.579589:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776651.579591:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776651.579593:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776651.579595:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776651.579600:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776651.579602:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776651.579604:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776651.579609:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776651.579611:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776651.579613:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775776651.579613:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776651.579617:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000010:2.0:1775776651.579617:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000100:00000001:0.0:1775776651.579619:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000040:2.0:1775776651.579620:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776651.579622:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776651.579623:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776651.579625:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449180 x1862022710430336/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776651.579625:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776651.579626:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776651.579628:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776651.579632:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775776651.579634:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776651.579639:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710430464/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776651.579646:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776651.579647:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776651.579651:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710430464/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776651.579658:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776651.579660:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776651.579663:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776651.579665:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776651.579666:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776651.579669:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776651.579671:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13534 00000400:00000200:2.0:1775776651.579675:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776651.579678:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776651.579680:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776651.579684:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776651.579685:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776651.579687:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776651.579689:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776651.579691:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776651.579693:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776651.579694:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776651.579696:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776651.579697:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000001:0.0:1775776651.579698:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776651.579700:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776651.579700:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776651.579703:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 02000000:00000001:0.0:1775776651.579704:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1775776651.579704:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1775776651.579706:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1775776651.579706:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776651.579707:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776651.579709:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 02000000:00000001:0.0:1775776651.579710:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776651.579712:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00100000:0.0:1775776651.579713:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710430336:192.168.201.115@tcp:250:kworker.0 00000100:00000001:2.0:1775776651.579714:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776651.579715:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776651.579718:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776651.579720:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775776651.579720:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776651.579722:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775776651.579724:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:2.0:1775776651.579724:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1775776651.579726:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775776651.579726:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776651.579728:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776651.579729:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:2.0:1775776651.579730:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775776651.579732:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776651.579734:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776651.579735:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776651.579737:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776651.579738:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776651.579740:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776651.579743:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 02000000:00000010:0.0:1775776651.579747:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7800. 00000100:00000001:2.0:1775776651.579747:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776651.579749:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776651.579751:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710430592/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775776651.579756:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:2.0:1775776651.579757:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776651.579759:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776651.579760:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000100:00000040:2.0:1775776651.579762:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710430592/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1775776651.579768:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0f68. 00000100:00000001:2.0:1775776651.579768:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776651.579771:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776651.579774:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000200:0.0:1775776651.579777:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710430336, portal 25 00000100:00000040:2.0:1775776651.579777:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776651.579779:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000040:2.0:1775776651.579780:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1775776651.579781:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662539648 : -123823047011968 : ffff8f6233449180) 00000100:00000040:2.0:1775776651.579785:0:141703:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13544) 00000100:00000040:0.0:1775776651.579786:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449180 x1862022710430336/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776676 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.579804:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776651.579807:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776651.579810:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776651.579812:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710430336, offset 0 00000400:00000200:0.0:1775776651.579820:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776651.579828:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776651.579835:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776651.579841:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776651.579846:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776651.579850:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776651.579852:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776651.579856:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453914:453914:256:4294967295] 192.168.201.115@tcp LPNI seq info [453839:453839:8:4294967295] 00000400:00000200:0.0:1775776651.579860:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776651.579873:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776651.579891:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776651.579899:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cb00. 00000800:00000200:0.0:1775776651.579907:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776651.579913:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775776651.579922:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776651.579926:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775776651.579931:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776651.579935:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cb00. 00000400:00000200:0.0:1775776651.579943:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776651.579949:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776651.579955:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776651.579957:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776651.579961:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776651.579965:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449180 x1862022710430336/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776676 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.579973:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776651.579976:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449180 x1862022710430336/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776676 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.579983:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776651.579985:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776651.579991:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776651.579996:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776651.579998:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308210438 00000100:00000010:0.0:1775776651.580003:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c00. 00000100:00000001:0.0:1775776651.580009:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776651.580016:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308210438 00000100:00000010:0.0:1775776651.580018:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3600. 00000100:00000001:0.0:1775776651.580023:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776651.580027:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776651.580029:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776651.580032:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776651.580036:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776651/real 1775776651] req@ffff8f6233449180 x1862022710430336/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776651.580045:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776651.580047:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776651.580050:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776651.580053:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0f68 00000400:00000010:0.0:1775776651.580054:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0f68. 00000100:00000001:0.0:1775776651.580057:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776651.580060:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449180 x1862022710430336/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776651.580069:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449180 x1862022710430336/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.580076:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776651.580078:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776651.580081:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776651.580083:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776651.580087:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449180 x1862022710430336/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.580095:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776651.580100:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449180 x1862022710430336/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.580108:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776651.580110:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776651.580112:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776651.580114:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776651.580117:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776651.580120:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776651.580123:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776651.580128:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776651.580131:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776651.580177:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449180 x1862022710430336/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776651.580186:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710430336:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776651.580194:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776651.580196:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776651.580198:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449180 x1862022710430336/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.580205:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776651.580209:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776651.580211:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776651.580214:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776651.580217:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776651.580218:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776651.580220:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776651.580223:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4cc00. 02000000:00000001:0.0:1775776651.580226:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776651.580228:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776651.580230:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449180. 00000100:00000001:0.0:1775776651.580233:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776651.580234:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776651.580236:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776651.580243:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776651.580245:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776651.580247:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776651.580248:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776651.580250:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776651.580253:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344ad80 x1862022710430464/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.580259:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776651.580260:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776651.580262:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776651.580264:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776651.580266:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776651.580269:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710430464:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776651.580272:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776651.580274:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776651.580275:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776651.580277:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776651.580281:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776651.580282:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776651.580287:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776651.580289:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776651.580293:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0f68. 00000100:00000200:0.0:1775776651.580296:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710430464, portal 10 00000100:00000001:0.0:1775776651.580298:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776651.580299:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662546816 : -123823047004800 : ffff8f623344ad80) 00000100:00000040:0.0:1775776651.580306:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344ad80 x1862022710430464/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776676 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.580313:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776651.580315:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776651.580317:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776651.580319:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710430464, offset 0 00000400:00000200:0.0:1775776651.580322:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776651.580327:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776651.580331:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776651.580334:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776651.580337:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776651.580339:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776651.580341:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776651.580344:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453915:453915:256:4294967295] 192.168.201.115@tcp LPNI seq info [453840:453840:8:4294967295] 00000400:00000200:0.0:1775776651.580348:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776651.580354:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776651.580358:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776651.580361:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cb00. 00000800:00000200:0.0:1775776651.580365:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776651.580366:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775776651.580369:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776651.580372:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775776651.580374:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776651.580376:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cb00. 00000400:00000200:0.0:1775776651.580379:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776651.580381:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776651.580384:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776651.580385:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776651.580387:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776651.580389:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344ad80 x1862022710430464/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776676 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.580395:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776651.580396:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344ad80 x1862022710430464/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776676 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.580401:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776651.580402:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776651.580404:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776651.580408:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776651.580410:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776651.580411:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308206604 00000100:00000001:0.0:1775776651.580413:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308210438 00000100:00000001:0.0:1775776651.580415:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308206604 00000100:00000001:0.0:1775776651.580417:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308210438 00000100:00000001:0.0:1775776651.580418:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776651.580419:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776651.580420:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776651.580422:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776651/real 1775776651] req@ffff8f623344ad80 x1862022710430464/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776651.580427:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776651.580428:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776651.580429:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776651.580431:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0f68 00000400:00000010:0.0:1775776651.580432:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0f68. 00000100:00000001:0.0:1775776651.580433:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776651.580435:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344ad80 x1862022710430464/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776651.580441:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344ad80 x1862022710430464/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.580448:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776651.580454:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776651.580456:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776651.580458:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776651.580462:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344ad80 x1862022710430464/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.580469:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776651.580473:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344ad80 x1862022710430464/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.580480:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776651.580482:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776651.580484:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776651.580485:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776651.580487:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776651.580492:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776651.580495:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776651.580498:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776651.580500:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776651.580504:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344ad80 x1862022710430464/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776651.580509:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710430464:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776651.580512:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776651.580514:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449500 x1862022710430592/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.580523:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776651.580524:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776651.580526:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776651.580528:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776651.580531:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776651.580533:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710430592:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776651.580537:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776651.580539:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776651.580541:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776651.580542:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776651.580546:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776651.580547:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776651.580551:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776651.580553:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776651.580556:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0f68. 00000100:00000200:0.0:1775776651.580559:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710430592, portal 10 00000100:00000001:0.0:1775776651.580561:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776651.580562:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662540544 : -123823047011072 : ffff8f6233449500) 00000100:00000040:0.0:1775776651.580568:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449500 x1862022710430592/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776676 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.580574:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776651.580577:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776651.580579:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776651.580581:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710430592, offset 0 00000400:00000200:0.0:1775776651.580585:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776651.580589:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776651.580593:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776651.580595:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776651.580597:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776651.580598:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776651.580600:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776651.580602:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453916:453916:256:4294967295] 192.168.201.115@tcp LPNI seq info [453841:453841:8:4294967295] 00000400:00000200:0.0:1775776651.580605:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776651.580609:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776651.580612:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776651.580614:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cb00. 00000800:00000200:0.0:1775776651.580616:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776651.580618:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1775776651.580621:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776651.580622:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1775776651.580624:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776651.580626:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cb00. 00000400:00000200:0.0:1775776651.580627:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776651.580630:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776651.580632:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776651.580633:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776651.580634:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776651.580643:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449500 x1862022710430592/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776676 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.580650:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776651.580653:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449500 x1862022710430592/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776676 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.580658:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776651.580660:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776651.580663:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776651.580666:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776651.580669:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776651.580671:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308206604 00000100:00000001:0.0:1775776651.580673:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308210438 00000100:00000001:0.0:1775776651.580675:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308206604 00000100:00000001:0.0:1775776651.580677:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308210438 00000100:00000001:0.0:1775776651.580680:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776651.580682:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776651.580684:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776651.580687:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776651/real 1775776651] req@ffff8f6233449500 x1862022710430592/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776651.580694:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776651.580695:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776651.580696:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776651.580697:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0f68 00000400:00000010:0.0:1775776651.580698:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0f68. 00000100:00000001:0.0:1775776651.580700:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776651.580702:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449500 x1862022710430592/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776651.580706:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449500 x1862022710430592/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.580710:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776651.580711:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776651.580712:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776651.580712:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776651.580714:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449500 x1862022710430592/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.580718:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776651.580720:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449500 x1862022710430592/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.580724:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776651.580725:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776651.580726:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776651.580727:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776651.580728:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776651.580729:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776651.580730:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776651.580733:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776651.580734:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776651.580737:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449500 x1862022710430592/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776651.580751:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710430592:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776651.580754:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776651.580756:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776651.580759:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344ad80 x1862022710430464/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.580766:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776651.580767:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776651.580769:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776651.580771:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776651.580773:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776651.580774:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776651.580776:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776651.580777:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4ec00. 02000000:00000001:0.0:1775776651.580779:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776651.580780:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776651.580782:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344ad80. 00000100:00000001:0.0:1775776651.580786:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776651.580797:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776651.580799:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776651.580803:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449500 x1862022710430592/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776676 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776651.580809:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776651.580810:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776651.580812:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776651.580814:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776651.580815:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776651.580816:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776651.580818:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776651.580819:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4f000. 02000000:00000001:0.0:1775776651.580821:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776651.580822:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776651.580823:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449500. 00000100:00000001:0.0:1775776651.580826:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776651.580827:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776651.580829:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776651.580832:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776651.580834:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776651.580835:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776651.580837:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776651.707290:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776651.707295:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776651.707301:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776651.707304:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776651.707307:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776651.707310:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776651.707312:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776651.707315:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776651.707316:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776651.707319:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776651.707323:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776651.707324:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776651.707325:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776651.707326:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776651.707327:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776651.707327:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776651.707329:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776651.707330:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776651.707548:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776651.707552:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776651.707555:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776652.603284:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776652.603293:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776652.603295:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776652.603302:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776652.603307:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776652.603312:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776652.603316:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776652.603319:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776652.603323:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776652.731315:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776652.731323:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776652.731331:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776652.731332:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776652.731336:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776652.731337:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776652.731399:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776652.731403:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776652.731406:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776653.627203:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776653.627211:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776653.627212:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776653.627218:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776653.627222:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776653.627226:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776653.627229:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776653.627231:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776653.627234:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776653.755265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776653.755273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776653.755280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776653.755282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776653.755286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776653.755287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776653.755357:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776653.755366:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776653.755370:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776654.651303:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776654.651313:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776654.651314:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776654.651321:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776654.651326:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776654.651330:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776654.651333:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776654.651336:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776654.651338:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776654.779243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776654.779250:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776654.779255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776654.779256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776654.779260:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776654.779261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776654.779333:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776654.779338:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776654.779341:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776655.483214:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308210438, at 4308210496 00000100:00000040:1.0:1775776655.483214:0:214731:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308210438, at 4308210496 00000100:00000010:1.0:1775776655.483226:0:214731:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3600. 00000100:00000010:0.0:1775776655.483229:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c00. 00000100:00000001:0.0:1775776655.675275:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776655.675282:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776655.675284:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776655.675289:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776655.675293:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776655.675297:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776655.675300:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776655.675302:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776655.675305:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776655.803265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776655.803270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776655.803276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776655.803277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776655.803280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776655.803281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776655.803404:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776655.803408:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776655.803411:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775776656.699244:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776656.699246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776656.699252:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1775776656.699252:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776656.699255:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775776656.699256:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776656.699258:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775776656.699261:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776656.699263:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:0.0:1775776656.699265:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776656.699269:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776656.699272:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776656.699274:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1775776656.699275:0:141703:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13539 00000100:00000001:0.0:1775776656.699279:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776656.699287:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776656.699294:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776656.699297:0:141703:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776656.699301:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776656.699304:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776656.699307:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776656.699311:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776656.699323:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776656.699325:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776656.699328:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776656.699331:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776656.699334:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776656.699336:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776656.699339:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776656.699341:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776656.699342:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776656.699344:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776656.699347:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776656.699350:0:141703:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776656.699352:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776656.699354:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776656.699356:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776656.699358:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776656.699359:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776656.699361:0:141703:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776656.699369:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775776656.699372:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776656.699375:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776656.699376:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776656.699378:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776656.699380:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776656.699383:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776656.699389:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775776656.699395:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776656.699404:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710430720/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776656.699413:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776656.699418:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776656.699423:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710430720/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776656.699443:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776656.699446:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776656.699450:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776656.699451:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776656.699452:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776656.699455:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776656.699457:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13539 00000400:00000200:2.0:1775776656.699462:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776656.699466:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776656.699468:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1775776656.699471:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1775776656.699471:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776656.699475:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1775776656.699476:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1775776656.699476:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776656.699479:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:0.0:1775776656.699480:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1775776656.699482:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776656.699484:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776656.699485:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776656.699487:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000040:0.0:1775776656.699489:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344aa00 x1862022710430720/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1775776656.699489:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776656.699493:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776656.699499:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776656.699502:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776656.699506:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776656.699510:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776656.699510:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:0.0:1775776656.699514:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1775776656.699514:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776656.699519:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 02000000:00000001:0.0:1775776656.699520:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1775776656.699522:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776656.699524:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776656.699527:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776656.699529:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776656.699534:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710430720:192.168.201.115@tcp:250:kworker.0 02000000:00000001:2.0:1775776656.699534:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776656.699539:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776656.699542:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1775776656.699543:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775776656.699546:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:2.0:1775776656.699548:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1775776656.699550:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775776656.699552:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776656.699554:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776656.699558:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 02000000:00000001:0.0:1775776656.699559:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:2.0:1775776656.699560:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776656.699563:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776656.699564:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776656.699565:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776656.699566:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:0.0:1775776656.699567:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:2.0:1775776656.699568:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776656.699573:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776656.699574:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775776656.699578:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776656.699581:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00100000:2.0:1775776656.699582:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710430848/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1775776656.699588:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000100:00000001:2.0:1775776656.699589:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776656.699592:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776656.699595:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710430848/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1775776656.699596:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0f68. 00000100:00000200:0.0:1775776656.699605:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710430720, portal 25 00000100:00000001:2.0:1775776656.699606:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776656.699611:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1775776656.699611:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776656.699614:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662545920 : -123823047005696 : ffff8f623344aa00) 00000100:00080000:2.0:1775776656.699616:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776656.699618:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776656.699619:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776656.699622:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776656.699626:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13539 00000100:00000040:0.0:1775776656.699631:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344aa00 x1862022710430720/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776681 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1775776656.699634:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776656.699641:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776656.699646:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776656.699652:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776656.699657:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776656.699661:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776656.699662:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1775776656.699665:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000040:0.0:1775776656.699669:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000020:00000001:2.0:1775776656.699670:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776656.699671:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000400:00000010:0.0:1775776656.699673:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000020:00000001:2.0:1775776656.699673:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776656.699675:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000100:00000200:0.0:1775776656.699678:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710430720, offset 0 00000020:00000001:2.0:1775776656.699678:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776656.699680:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776656.699684:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776656.699685:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776656.699687:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776656.699690:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:0.0:1775776656.699691:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000020:00000040:2.0:1775776656.699692:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776656.699696:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776656.699713:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776656.699716:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:2.0:1775776656.699716:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776656.699720:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776656.699722:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776656.699723:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:0.0:1775776656.699726:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000008:00100000:2.0:1775776656.699726:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776656.699731:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776656.699732:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776656.699734:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000010:2.0:1775776656.699735:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775776656.699737:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000400:00000200:0.0:1775776656.699740:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 02000000:00000001:2.0:1775776656.699740:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776656.699742:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776656.699744:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776656.699746:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:0.0:1775776656.699747:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 02000000:00000001:2.0:1775776656.699748:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776656.699753:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 02000000:00000010:2.0:1775776656.699754:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000400:00000200:0.0:1775776656.699759:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453917:453917:256:4294967295] 192.168.201.115@tcp LPNI seq info [453842:453842:8:4294967295] 00000100:00000001:2.0:1775776656.699759:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776656.699765:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710430976/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775776656.699769:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:2.0:1775776656.699776:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776656.699778:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776656.699782:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710430976/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776656.699789:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:2.0:1775776656.699795:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776656.699798:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.0:1775776656.699801:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000040:2.0:1775776656.699803:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776656.699806:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776656.699808:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000010:0.0:1775776656.699809:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cb00. 00000100:00000040:2.0:1775776656.699811:0:141703:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13549) 00000800:00000200:0.0:1775776656.699818:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776656.699822:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775776656.699828:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776656.699831:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1775776656.699836:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776656.699838:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cb00. 00000400:00000200:0.0:1775776656.699843:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776656.699846:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776656.699850:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776656.699852:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776656.699855:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776656.699858:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344aa00 x1862022710430720/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776681 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.699865:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776656.699867:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344aa00 x1862022710430720/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776681 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.699880:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776656.699881:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776656.699886:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776656.699890:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776656.699892:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308215558 00000100:00000010:0.0:1775776656.699897:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c00. 00000100:00000001:0.0:1775776656.699903:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776656.699908:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308215558 00000100:00000010:0.0:1775776656.699911:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c60. 00000100:00000001:0.0:1775776656.699914:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776656.699918:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776656.699920:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776656.699923:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776656.699927:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776656/real 1775776656] req@ffff8f623344aa00 x1862022710430720/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776656.699935:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776656.699936:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776656.699937:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776656.699939:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0f68 00000400:00000010:0.0:1775776656.699940:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0f68. 00000100:00000001:0.0:1775776656.699941:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776656.699944:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344aa00 x1862022710430720/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776656.699949:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344aa00 x1862022710430720/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.699953:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776656.699954:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776656.699956:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776656.699957:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776656.699959:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344aa00 x1862022710430720/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.699964:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776656.699966:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344aa00 x1862022710430720/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.699970:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776656.699972:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776656.699973:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776656.699973:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776656.699975:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776656.699976:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776656.699978:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776656.699980:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776656.699981:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776656.699983:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344aa00 x1862022710430720/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776656.699988:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710430720:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776656.699990:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776656.699991:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776656.699993:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344aa00 x1862022710430720/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.699997:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776656.699998:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776656.700000:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776656.700001:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776656.700002:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776656.700003:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776656.700005:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776656.700006:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4e800. 02000000:00000001:0.0:1775776656.700009:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776656.700010:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776656.700011:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344aa00. 00000100:00000001:0.0:1775776656.700014:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776656.700015:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776656.700016:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776656.700021:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776656.700022:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776656.700024:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776656.700025:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776656.700025:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776656.700027:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233448380 x1862022710430848/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.700031:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776656.700032:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776656.700034:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776656.700035:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776656.700036:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776656.700037:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233448380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710430848:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776656.700039:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776656.700040:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776656.700041:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776656.700042:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776656.700044:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776656.700045:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776656.700047:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776656.700048:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776656.700051:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0f68. 00000100:00000200:0.0:1775776656.700052:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710430848, portal 10 00000100:00000001:0.0:1775776656.700054:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776656.700055:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662536064 : -123823047015552 : ffff8f6233448380) 00000100:00000040:0.0:1775776656.700058:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233448380 x1862022710430848/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776681 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.700062:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776656.700064:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776656.700065:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776656.700067:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710430848, offset 0 00000400:00000200:0.0:1775776656.700069:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776656.700071:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776656.700074:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776656.700076:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776656.700077:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776656.700079:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776656.700080:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776656.700082:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453918:453918:256:4294967295] 192.168.201.115@tcp LPNI seq info [453843:453843:8:4294967295] 00000400:00000200:0.0:1775776656.700084:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776656.700088:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776656.700090:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776656.700092:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cb00. 00000800:00000200:0.0:1775776656.700094:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776656.700096:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775776656.700098:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776656.700100:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1775776656.700102:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776656.700103:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cb00. 00000400:00000200:0.0:1775776656.700105:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776656.700107:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776656.700108:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776656.700109:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776656.700111:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776656.700113:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233448380 x1862022710430848/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776681 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.700117:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776656.700118:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448380 x1862022710430848/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776681 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.700122:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776656.700123:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776656.700125:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776656.700128:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776656.700129:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776656.700130:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308211723 00000100:00000001:0.0:1775776656.700132:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308215558 00000100:00000001:0.0:1775776656.700151:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308211724 00000100:00000001:0.0:1775776656.700152:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308215558 00000100:00000001:0.0:1775776656.700154:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776656.700155:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776656.700156:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776656.700159:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776656/real 1775776656] req@ffff8f6233448380 x1862022710430848/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776656.700164:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776656.700164:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776656.700165:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776656.700167:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0f68 00000400:00000010:0.0:1775776656.700168:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0f68. 00000100:00000001:0.0:1775776656.700169:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776656.700171:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233448380 x1862022710430848/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776656.700176:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233448380 x1862022710430848/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.700179:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776656.700180:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776656.700181:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776656.700182:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776656.700183:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233448380 x1862022710430848/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.700187:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776656.700189:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233448380 x1862022710430848/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.700193:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776656.700194:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776656.700195:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776656.700196:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776656.700197:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776656.700200:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776656.700201:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776656.700203:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776656.700204:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776656.700206:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233448380 x1862022710430848/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776656.700211:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233448380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710430848:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776656.700213:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776656.700214:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449f80 x1862022710430976/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.700218:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776656.700219:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776656.700220:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776656.700221:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776656.700222:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776656.700223:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710430976:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776656.700225:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776656.700226:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776656.700226:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776656.700227:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776656.700230:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776656.700231:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776656.700233:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776656.700234:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776656.700236:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0f68. 00000100:00000200:0.0:1775776656.700238:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710430976, portal 10 00000100:00000001:0.0:1775776656.700239:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776656.700239:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662543232 : -123823047008384 : ffff8f6233449f80) 00000100:00000040:0.0:1775776656.700242:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449f80 x1862022710430976/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776681 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.700246:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776656.700247:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776656.700248:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776656.700250:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710430976, offset 0 00000400:00000200:0.0:1775776656.700252:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776656.700254:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776656.700255:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776656.700257:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776656.700258:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776656.700259:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776656.700261:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776656.700263:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453919:453919:256:4294967295] 192.168.201.115@tcp LPNI seq info [453844:453844:8:4294967295] 00000400:00000200:0.0:1775776656.700265:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776656.700269:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776656.700271:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776656.700273:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cb00. 00000800:00000200:0.0:1775776656.700275:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776656.700276:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1775776656.700279:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776656.700280:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 53 secs later) 00000800:00000100:0.0:1775776656.700282:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776656.700284:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cb00. 00000400:00000200:0.0:1775776656.700285:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776656.700287:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776656.700289:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776656.700290:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776656.700291:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776656.700293:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449f80 x1862022710430976/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776681 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.700297:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776656.700299:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449f80 x1862022710430976/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776681 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.700302:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776656.700303:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776656.700305:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776656.700306:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776656.700308:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776656.700309:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308211724 00000100:00000001:0.0:1775776656.700311:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308215558 00000100:00000001:0.0:1775776656.700312:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308211724 00000100:00000001:0.0:1775776656.700313:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308215558 00000100:00000001:0.0:1775776656.700314:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776656.700315:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776656.700316:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776656.700318:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776656/real 1775776656] req@ffff8f6233449f80 x1862022710430976/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776656.700322:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776656.700322:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776656.700323:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776656.700325:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0f68 00000400:00000010:0.0:1775776656.700325:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0f68. 00000100:00000001:0.0:1775776656.700327:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776656.700328:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449f80 x1862022710430976/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776656.700332:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449f80 x1862022710430976/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.700336:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776656.700337:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776656.700338:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776656.700339:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776656.700340:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449f80 x1862022710430976/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.700351:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776656.700353:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449f80 x1862022710430976/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.700357:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776656.700358:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776656.700359:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776656.700360:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776656.700361:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776656.700362:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776656.700363:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776656.700365:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776656.700366:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776656.700368:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449f80 x1862022710430976/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776656.700372:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710430976:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776656.700373:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776656.700374:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776656.700376:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233448380 x1862022710430848/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.700380:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776656.700381:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776656.700382:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776656.700384:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776656.700384:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776656.700385:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776656.700386:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776656.700387:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4f400. 02000000:00000001:0.0:1775776656.700389:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776656.700389:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776656.700390:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233448380. 00000100:00000001:0.0:1775776656.700393:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776656.700394:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776656.700395:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776656.700398:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449f80 x1862022710430976/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776681 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776656.700402:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776656.700402:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776656.700403:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776656.700405:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776656.700405:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776656.700406:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776656.700407:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776656.700408:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4c000. 02000000:00000001:0.0:1775776656.700409:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776656.700409:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776656.700410:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449f80. 00000100:00000001:0.0:1775776656.700411:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776656.700412:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776656.700413:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776656.700415:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776656.700415:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776656.700416:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776656.700417:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776656.827242:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776656.827247:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776656.827249:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776656.827252:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776656.827256:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776656.827258:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776656.827260:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776656.827262:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776656.827265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776656.827267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776656.827271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776656.827272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776656.827337:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776656.827341:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776656.827344:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776657.723268:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776657.723274:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776657.723275:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776657.723279:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776657.723283:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776657.723286:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776657.723288:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776657.723290:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776657.723292:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776657.851267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776657.851274:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776657.851291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776657.851293:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776657.851297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776657.851298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776657.851360:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776657.851364:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776657.851368:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776658.747256:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776658.747262:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776658.747263:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776658.747268:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776658.747272:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776658.747276:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776658.747279:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776658.747281:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776658.747284:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776658.875255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776658.875260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776658.875266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776658.875267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776658.875271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776658.875272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776658.875337:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776658.875341:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776658.875344:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776659.771217:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776659.771222:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776659.771224:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776659.771228:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776659.771231:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776659.771234:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776659.771237:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776659.771239:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776659.771241:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776659.899279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776659.899285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776659.899296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776659.899298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776659.899302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776659.899303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776659.899361:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776659.899365:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776659.899368:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776660.603256:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308215558, at 4308215616 00000100:00000040:1.0:1775776660.603257:0:214731:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308215558, at 4308215616 00000100:00000010:1.0:1775776660.603287:0:214731:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c60. 00000100:00000010:0.0:1775776660.603291:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c00. 00000100:00000001:0.0:1775776660.795298:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776660.795307:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776660.795309:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776660.795317:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776660.795322:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776660.795327:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776660.795330:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776660.795333:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776660.795336:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776660.923279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776660.923288:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776660.923296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776660.923297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776660.923302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776660.923303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776660.923380:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776660.923390:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776660.923395:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776661.819218:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776661.819229:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1775776661.819230:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776661.819231:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776661.819241:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:2.0:1775776661.819241:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776661.819246:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776661.819246:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1775776661.819253:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776661.819256:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776661.819259:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776661.819262:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776661.819263:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776661.819269:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776661.819279:0:141703:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13544 00000400:00000200:2.0:1775776661.819292:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776661.819301:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776661.819305:0:141703:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776661.819308:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776661.819312:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776661.819315:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776661.819319:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776661.819323:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776661.819325:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776661.819338:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776661.819342:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776661.819345:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776661.819347:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776661.819350:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776661.819352:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776661.819354:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776661.819356:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776661.819359:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776661.819363:0:141703:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776661.819366:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776661.819370:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776661.819374:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776661.819378:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776661.819381:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776661.819393:0:141703:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776661.819405:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344a680. 00000020:00000040:2.0:1775776661.819410:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776661.819414:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776661.819416:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776661.819419:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776661.819422:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776661.819426:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776661.819437:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775776661.819446:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776661.819457:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344a680 x1862022710431104/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776661.819468:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776661.819474:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776661.819479:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006a4a647e] to pc [ptlrpcd_rcv+-1] req@ffff8f623344a680 x1862022710431104/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776661.819512:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776661.819515:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776661.819518:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776661.819519:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776661.819520:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776661.819524:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776661.819526:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13544 00000400:00000200:2.0:1775776661.819530:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776661.819532:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776661.819534:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776661.819537:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776661.819539:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776661.819541:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776661.819542:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776661.819544:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776661.819546:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776661.819547:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776661.819549:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776661.819552:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776661.819554:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776661.819557:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776661.819558:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776661.819560:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776661.819562:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776661.819564:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776661.819566:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776661.819569:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776661.819571:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776661.819584:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776661.819587:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776661.819590:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776661.819595:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:2.0:1775776661.819595:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1775776661.819600:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000008:00000001:2.0:1775776661.819600:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776661.819604:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1775776661.819605:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776661.819627:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449f80. 00000020:00000040:2.0:1775776661.819630:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776661.819633:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776661.819634:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776661.819636:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776661.819637:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776661.819640:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776661.819642:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344a680 x1862022710431104/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1775776661.819645:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000100:00000001:2.0:1775776661.819651:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776661.819656:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776661.819658:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776661.819661:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449f80 x1862022710431232/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1775776661.819664:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776661.819668:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776661.819672:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776661.819674:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776661.819678:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a7feaad7] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449f80 x1862022710431232/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775776661.819685:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776661.819693:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710431104:192.168.201.115@tcp:250:kworker.0 00000100:00000001:2.0:1775776661.819693:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776661.819697:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1775776661.819701:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1775776661.819702:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776661.819705:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776661.819706:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1775776661.819709:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1775776661.819710:0:141703:0:(import.c:484:import_select_connection()) Process entered 02000000:00000001:0.0:1775776661.819713:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776661.819714:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13544 02000000:00000001:0.0:1775776661.819718:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:2.0:1775776661.819721:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776661.819724:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 02000000:00000010:0.0:1775776661.819725:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7800. 00000100:00080000:2.0:1775776661.819728:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1775776661.819729:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776661.819734:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776661.819737:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775776661.819738:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:2.0:1775776661.819741:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000010:0.0:1775776661.819745:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000100:00000001:2.0:1775776661.819745:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776661.819749:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776661.819751:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776661.819753:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776661.819755:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000400:00000010:0.0:1775776661.819756:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0f68. 00000020:00000001:2.0:1775776661.819757:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776661.819761:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776661.819764:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000200:0.0:1775776661.819767:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710431104, portal 25 00000100:00000001:2.0:1775776661.819767:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776661.819770:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776661.819772:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776661.819774:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:0.0:1775776661.819775:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662545024 : -123823047006592 : ffff8f623344a680) 00000020:00000040:2.0:1775776661.819778:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00000040:0.0:1775776661.819784:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344a680 x1862022710431104/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776686 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775776661.819784:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776661.819790:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776661.819795:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776661.819798:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776661.819799:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1775776661.819803:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776661.819805:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:2.0:1775776661.819807:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776661.819810:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776661.819812:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776661.819814:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776661.819818:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448380. 00000400:00000010:0.0:1775776661.819819:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000020:00000040:2.0:1775776661.819820:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000100:00000200:0.0:1775776661.819825:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710431104, offset 0 02000000:00000001:2.0:1775776661.819825:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776661.819827:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776661.819830:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776661.819831:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776661.819833:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776661.819836:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 02000000:00000010:2.0:1775776661.819837:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000100:00000001:2.0:1775776661.819842:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776661.819846:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776661.819852:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00100000:2.0:1775776661.819858:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448380 x1862022710431360/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775776661.819859:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776661.819867:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:2.0:1775776661.819869:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776661.819872:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776661.819874:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776661.819880:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000040:2.0:1775776661.819880:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bc1bdc07] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448380 x1862022710431360/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776661.819890:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453920:453920:256:4294967295] 192.168.201.115@tcp LPNI seq info [453845:453845:8:4294967295] 00000100:00000001:2.0:1775776661.819894:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776661.819899:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1775776661.819900:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:2.0:1775776661.819905:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776661.819909:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776661.819912:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776661.819917:0:141703:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13554) 00000400:00000200:0.0:1775776661.819928:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776661.819937:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776661.819944:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cb00. 00000800:00000200:0.0:1775776661.819953:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776661.819957:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775776661.819963:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776661.819967:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1775776661.819972:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776661.819974:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cb00. 00000400:00000200:0.0:1775776661.819982:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776661.819993:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776661.820000:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776661.820003:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776661.820008:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776661.820013:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344a680 x1862022710431104/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776686 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820022:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776661.820026:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344a680 x1862022710431104/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776686 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820034:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776661.820036:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776661.820043:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776661.820048:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776661.820051:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308220678 00000100:00000010:0.0:1775776661.820057:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c00. 00000100:00000001:0.0:1775776661.820066:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776661.820073:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308220678 00000100:00000010:0.0:1775776661.820083:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3840. 00000100:00000001:0.0:1775776661.820086:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776661.820090:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776661.820093:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776661.820096:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776661.820100:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776661/real 1775776661] req@ffff8f623344a680 x1862022710431104/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776661.820110:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776661.820112:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776661.820114:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776661.820117:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0f68 00000400:00000010:0.0:1775776661.820119:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0f68. 00000100:00000001:0.0:1775776661.820121:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776661.820125:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344a680 x1862022710431104/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776661.820150:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344a680 x1862022710431104/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820163:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776661.820165:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776661.820168:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776661.820171:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776661.820174:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344a680 x1862022710431104/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820182:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776661.820186:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344a680 x1862022710431104/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820193:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776661.820196:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776661.820198:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776661.820199:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776661.820202:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776661.820210:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776661.820214:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776661.820218:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776661.820221:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776661.820225:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344a680 x1862022710431104/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776661.820231:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710431104:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776661.820234:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776661.820236:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776661.820237:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344a680 x1862022710431104/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820242:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776661.820244:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776661.820246:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776661.820248:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776661.820249:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776661.820250:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776661.820251:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776661.820253:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4e000. 02000000:00000001:0.0:1775776661.820257:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776661.820259:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776661.820260:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344a680. 00000100:00000001:0.0:1775776661.820265:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776661.820266:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776661.820268:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776661.820276:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776661.820278:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776661.820280:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776661.820281:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776661.820282:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776661.820286:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449f80 x1862022710431232/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820293:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776661.820294:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776661.820296:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776661.820298:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776661.820300:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776661.820309:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710431232:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776661.820313:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776661.820315:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776661.820316:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776661.820318:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776661.820321:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776661.820323:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776661.820326:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776661.820329:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776661.820333:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0f68. 00000100:00000200:0.0:1775776661.820335:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710431232, portal 10 00000100:00000001:0.0:1775776661.820336:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776661.820337:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662543232 : -123823047008384 : ffff8f6233449f80) 00000100:00000040:0.0:1775776661.820343:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449f80 x1862022710431232/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776686 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820347:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776661.820348:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776661.820350:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776661.820351:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710431232, offset 0 00000400:00000200:0.0:1775776661.820354:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776661.820357:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776661.820360:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776661.820362:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776661.820363:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776661.820365:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776661.820367:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776661.820369:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453921:453921:256:4294967295] 192.168.201.115@tcp LPNI seq info [453846:453846:8:4294967295] 00000400:00000200:0.0:1775776661.820371:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776661.820375:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776661.820378:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776661.820380:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cb00. 00000800:00000200:0.0:1775776661.820382:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776661.820384:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775776661.820388:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776661.820390:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1775776661.820392:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776661.820393:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cb00. 00000400:00000200:0.0:1775776661.820395:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776661.820397:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776661.820399:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776661.820400:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776661.820402:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776661.820404:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449f80 x1862022710431232/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776686 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820410:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776661.820412:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449f80 x1862022710431232/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776686 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820417:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776661.820418:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776661.820420:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776661.820424:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776661.820426:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776661.820427:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308216844 00000100:00000001:0.0:1775776661.820429:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308220678 00000100:00000001:0.0:1775776661.820431:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308216844 00000100:00000001:0.0:1775776661.820432:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308220678 00000100:00000001:0.0:1775776661.820434:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776661.820435:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776661.820437:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776661.820438:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776661/real 1775776661] req@ffff8f6233449f80 x1862022710431232/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776661.820443:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776661.820444:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776661.820445:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776661.820446:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0f68 00000400:00000010:0.0:1775776661.820447:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0f68. 00000100:00000001:0.0:1775776661.820449:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776661.820450:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449f80 x1862022710431232/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776661.820455:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449f80 x1862022710431232/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820459:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776661.820460:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776661.820461:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776661.820461:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776661.820463:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449f80 x1862022710431232/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820467:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776661.820468:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449f80 x1862022710431232/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820473:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776661.820475:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776661.820476:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776661.820476:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776661.820477:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776661.820481:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776661.820482:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776661.820484:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776661.820485:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776661.820487:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449f80 x1862022710431232/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776661.820492:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710431232:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776661.820494:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776661.820496:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233448380 x1862022710431360/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820499:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776661.820500:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776661.820501:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776661.820502:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776661.820503:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776661.820504:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233448380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710431360:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776661.820506:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776661.820507:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776661.820508:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776661.820509:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776661.820511:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776661.820512:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776661.820514:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776661.820515:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776661.820517:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0f68. 00000100:00000200:0.0:1775776661.820519:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710431360, portal 10 00000100:00000001:0.0:1775776661.820520:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776661.820521:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662536064 : -123823047015552 : ffff8f6233448380) 00000100:00000040:0.0:1775776661.820524:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233448380 x1862022710431360/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776686 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820528:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776661.820529:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776661.820530:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776661.820531:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710431360, offset 0 00000400:00000200:0.0:1775776661.820533:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776661.820535:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776661.820537:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776661.820538:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776661.820540:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776661.820541:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776661.820543:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776661.820544:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453922:453922:256:4294967295] 192.168.201.115@tcp LPNI seq info [453847:453847:8:4294967295] 00000400:00000200:0.0:1775776661.820547:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776661.820550:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776661.820552:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776661.820554:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cb00. 00000800:00000200:0.0:1775776661.820556:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776661.820558:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1775776661.820560:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776661.820562:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 48 secs later) 00000800:00000100:0.0:1775776661.820564:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776661.820565:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cb00. 00000400:00000200:0.0:1775776661.820567:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776661.820569:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776661.820570:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776661.820572:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776661.820573:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776661.820575:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233448380 x1862022710431360/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776686 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820580:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776661.820581:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448380 x1862022710431360/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776686 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820585:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776661.820585:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776661.820587:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776661.820589:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776661.820590:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776661.820592:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308216844 00000100:00000001:0.0:1775776661.820593:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308220678 00000100:00000001:0.0:1775776661.820595:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308216844 00000100:00000001:0.0:1775776661.820596:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308220678 00000100:00000001:0.0:1775776661.820597:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776661.820598:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776661.820599:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776661.820601:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776661/real 1775776661] req@ffff8f6233448380 x1862022710431360/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776661.820604:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776661.820605:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776661.820606:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776661.820607:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0f68 00000400:00000010:0.0:1775776661.820619:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0f68. 00000100:00000001:0.0:1775776661.820622:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776661.820626:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233448380 x1862022710431360/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776661.820637:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233448380 x1862022710431360/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820643:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776661.820645:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776661.820646:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776661.820647:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776661.820651:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233448380 x1862022710431360/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820657:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776661.820661:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233448380 x1862022710431360/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820667:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776661.820669:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776661.820670:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776661.820671:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776661.820673:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776661.820675:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776661.820677:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776661.820680:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776661.820682:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776661.820685:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233448380 x1862022710431360/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776661.820691:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233448380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710431360:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776661.820695:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776661.820697:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776661.820699:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449f80 x1862022710431232/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820705:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776661.820707:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776661.820709:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776661.820711:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776661.820713:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776661.820714:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776661.820716:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776661.820718:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4f000. 02000000:00000001:0.0:1775776661.820720:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776661.820721:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776661.820722:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449f80. 00000100:00000001:0.0:1775776661.820724:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776661.820725:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776661.820726:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776661.820728:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233448380 x1862022710431360/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776686 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776661.820732:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776661.820733:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776661.820733:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776661.820735:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776661.820735:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776661.820736:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776661.820737:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776661.820738:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4ec00. 02000000:00000001:0.0:1775776661.820739:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776661.820739:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776661.820740:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233448380. 00000100:00000001:0.0:1775776661.820741:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776661.820742:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776661.820743:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776661.820745:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776661.820746:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776661.820747:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776661.820748:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776661.947296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776661.947304:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776661.947310:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776661.947314:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776661.947316:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776661.947321:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776661.947324:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776661.947327:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776661.947329:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776661.947331:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776661.947336:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776661.947337:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776661.947338:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776661.947340:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776661.947342:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776661.947343:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776661.947344:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776661.947346:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776661.947457:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776661.947461:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776661.947465:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776662.844002:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776662.844012:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776662.844014:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776662.844021:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776662.844027:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776662.844032:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776662.844035:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776662.844038:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776662.844041:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776662.971247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776662.971256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776662.971263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776662.971265:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776662.971269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776662.971270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775776662.971335:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775776662.971339:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775776662.971343:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776663.867312:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776663.867321:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776663.867322:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776663.867330:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776663.867335:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776663.867340:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776663.867343:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776663.867345:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776663.867348:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776663.995263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776663.995269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776663.995275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776663.995276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776663.995280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776663.995281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776663.995343:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776663.995347:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776663.995350:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776664.891518:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776664.891523:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776664.891524:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776664.891528:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776664.891531:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776664.891534:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776664.891536:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776664.891538:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776664.891539:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776665.019234:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776665.019238:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776665.019241:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776665.019242:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776665.019246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776665.019247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776665.019304:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776665.019308:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776665.019312:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776665.723217:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308220678, at 4308220736 00000100:00000010:0.0:1775776665.723228:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c00. 00000100:00000040:1.0:1775776665.725660:0:214731:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308220678, at 4308220738 00000100:00000010:1.0:1775776665.725672:0:214731:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3840. 00000100:00000001:0.0:1775776665.915321:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776665.915328:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776665.915330:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776665.915335:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776665.915339:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776665.915343:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776665.915346:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776665.915348:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776665.915351:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776666.043297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776666.043302:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776666.043307:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776666.043308:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776666.043312:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776666.043313:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776666.043380:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776666.043389:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776666.043393:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776666.939251:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776666.939268:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776666.939270:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776666.939275:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776666.939279:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776666.939283:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776666.939286:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776666.939289:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776666.939291:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776666.939308:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776666.939315:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1775776666.939318:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776666.939322:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776666.939326:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776666.939332:0:141703:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13549 00000400:00000200:3.0:1775776666.939340:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776666.939346:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776666.939349:0:141703:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776666.939352:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776666.939355:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776666.939357:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776666.939359:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776666.939363:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776666.939365:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776666.939367:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776666.939369:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776666.939371:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776666.939373:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776666.939376:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776666.939378:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776666.939380:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776666.939381:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776666.939384:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776666.939388:0:141703:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776666.939390:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776666.939393:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776666.939396:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776666.939399:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776666.939401:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776666.939404:0:141703:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776666.939412:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71500. 00000020:00000040:3.0:1775776666.939425:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776666.939429:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776666.939431:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776666.939556:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776666.939559:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776666.939563:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776666.939571:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000100:00000001:3.0:1775776666.939577:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776666.939585:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71500 x1862022710431488/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776666.939595:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776666.939599:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776666.939603:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dff687b3] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71500 x1862022710431488/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776666.939627:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776666.939630:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776666.939634:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776666.939635:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776666.939637:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776666.939639:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776666.939642:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13549 00000400:00000200:3.0:1775776666.939646:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776666.939649:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776666.939652:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776666.939656:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776666.939658:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776666.939660:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776666.939662:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776666.939665:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776666.939667:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776666.939668:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:0.0:1775776666.939670:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1775776666.939670:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:0.0:1775776666.939674:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1775776666.939674:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776666.939677:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:0.0:1775776666.939678:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1775776666.939682:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776666.939685:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776666.939688:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71500 x1862022710431488/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776666.939689:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776666.939693:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776666.939698:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776666.939703:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776666.939708:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776666.939708:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776666.939711:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776666.939712:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776666.939717:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1775776666.939718:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776666.939721:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776666.939721:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776666.939726:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:0.0:1775776666.939727:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1775776666.939731:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775776666.939733:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710431488:192.168.201.115@tcp:250:kworker.0 00000008:00000001:3.0:1775776666.939735:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776666.939740:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1775776666.939740:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776666.939744:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70a80. 02000000:00000001:0.0:1775776666.939745:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:3.0:1775776666.939748:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776666.939749:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776666.939753:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776666.939754:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1775776666.939755:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776666.939758:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000010:0.0:1775776666.939760:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776666.939764:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776666.939769:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:3.0:1775776666.939769:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776666.939772:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1775776666.939773:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 02000000:00000010:3.0:1775776666.939778:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000400:00000010:0.0:1775776666.939781:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0f68. 00000100:00000001:3.0:1775776666.939783:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1775776666.939790:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710431488, portal 25 00000100:00100000:3.0:1775776666.939790:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70a80 x1862022710431616/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776666.939795:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776666.939798:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849009408 : -123823860542208 : ffff8f6202c71500) 00000100:00000001:3.0:1775776666.939803:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776666.939807:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776666.939809:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71500 x1862022710431488/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776691 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776666.939813:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000455feef6] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70a80 x1862022710431616/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.939827:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775776666.939832:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776666.939833:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00080000:3.0:1775776666.939836:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:0.0:1775776666.939837:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00080000:3.0:1775776666.939842:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000200:0.0:1775776666.939844:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710431488, offset 0 00000100:00000001:3.0:1775776666.939844:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776666.939847:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776666.939850:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:0.0:1775776666.939854:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00080000:3.0:1775776666.939855:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13549 00000400:00000200:3.0:1775776666.939861:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776666.939864:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776666.939867:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776666.939870:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1775776666.939873:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000040:3.0:1775776666.939877:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776666.939880:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775776666.939881:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1775776666.939884:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776666.939887:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:0.0:1775776666.939888:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000020:00000001:3.0:1775776666.939891:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776666.939893:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000400:00000200:0.0:1775776666.939894:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000020:00000001:3.0:1775776666.939896:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776666.939899:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000400:00000200:0.0:1775776666.939901:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000020:00000001:3.0:1775776666.939903:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776666.939907:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:0.0:1775776666.939908:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453923:453923:256:4294967295] 192.168.201.115@tcp LPNI seq info [453848:453848:8:4294967295] 00000100:00000040:3.0:1775776666.939913:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776666.939917:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1775776666.939918:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:3.0:1775776666.939921:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776666.939924:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776666.939927:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:3.0:1775776666.939932:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776666.939937:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776666.939938:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 02000000:00000001:3.0:1775776666.939940:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776666.939944:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776666.939948:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1775776666.939949:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776666.939952:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776666.939955:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000010:0.0:1775776666.939956:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cb00. 00000008:00000001:3.0:1775776666.939958:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776666.939961:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776666.939965:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c70700. 00000800:00000200:0.0:1775776666.939966:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000020:00000040:3.0:1775776666.939970:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000800:00000200:0.0:1775776666.939971:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 02000000:00000001:3.0:1775776666.939974:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776666.939977:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776666.939981:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:0.1:1775776666.939981:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000001:3.0:1775776666.939983:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776666.939986:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1775776666.939987:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 02000000:00000010:3.0:1775776666.939990:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000800:00000100:0.0:1775776666.939995:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776666.939996:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1775776666.939998:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cb00. 00000100:00100000:3.0:1775776666.940003:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c70700 x1862022710431744/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1775776666.940005:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776666.940015:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1775776666.940015:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776666.940019:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776666.940024:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd1061a5] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c70700 x1862022710431744/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776666.940028:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776666.940032:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:3.0:1775776666.940036:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776666.940040:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1775776666.940040:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776666.940046:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000200:0.0:1775776666.940047:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71500 x1862022710431488/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776691 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1775776666.940050:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776666.940059:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776666.940065:0:141703:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13559) 00000100:00000001:0.0:1775776666.940067:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776666.940072:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71500 x1862022710431488/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776691 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940080:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776666.940082:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776666.940088:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776666.940093:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776666.940096:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308225798 00000100:00000010:0.0:1775776666.940100:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c3c00. 00000100:00000001:0.0:1775776666.940106:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775776666.940111:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308225798 00000100:00000010:0.0:1775776666.940114:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62264c36c0. 00000100:00000001:0.0:1775776666.940117:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775776666.940121:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776666.940123:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776666.940125:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776666.940129:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776666/real 1775776666] req@ffff8f6202c71500 x1862022710431488/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776666.940159:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776666.940161:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776666.940163:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776666.940165:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0f68 00000400:00000010:0.0:1775776666.940167:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0f68. 00000100:00000001:0.0:1775776666.940169:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776666.940173:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71500 x1862022710431488/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776666.940181:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71500 x1862022710431488/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940187:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776666.940188:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776666.940191:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776666.940192:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776666.940196:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71500 x1862022710431488/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940203:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776666.940207:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71500 x1862022710431488/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940215:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776666.940217:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776666.940219:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776666.940220:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776666.940222:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776666.940225:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776666.940227:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776666.940231:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776666.940233:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776666.940237:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71500 x1862022710431488/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776666.940245:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710431488:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776666.940250:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776666.940252:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776666.940267:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71500 x1862022710431488/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940273:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776666.940275:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776666.940277:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776666.940280:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776666.940281:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776666.940283:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:0.0:1775776666.940285:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776666.940287:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35c00. 02000000:00000001:0.0:1775776666.940290:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776666.940292:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776666.940293:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71500. 00000100:00000001:0.0:1775776666.940296:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776666.940297:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776666.940299:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776666.940306:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776666.940307:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776666.940309:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776666.940319:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776666.940321:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776666.940325:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70a80 x1862022710431616/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940331:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776666.940333:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776666.940335:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776666.940336:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776666.940338:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776666.940341:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710431616:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776666.940344:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776666.940346:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776666.940347:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776666.940349:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776666.940352:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776666.940354:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776666.940357:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776666.940360:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776666.940364:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0f68. 00000100:00000200:0.0:1775776666.940367:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710431616, portal 10 00000100:00000001:0.0:1775776666.940369:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776666.940370:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849006720 : -123823860544896 : ffff8f6202c70a80) 00000100:00000040:0.0:1775776666.940376:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70a80 x1862022710431616/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776691 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940382:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776666.940385:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776666.940388:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776666.940389:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710431616, offset 0 00000400:00000200:0.0:1775776666.940393:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776666.940397:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776666.940400:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776666.940403:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776666.940406:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776666.940408:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776666.940411:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776666.940414:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453924:453924:256:4294967295] 192.168.201.115@tcp LPNI seq info [453849:453849:8:4294967295] 00000400:00000200:0.0:1775776666.940418:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776666.940425:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776666.940429:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776666.940433:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cb00. 00000800:00000200:0.0:1775776666.940437:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776666.940439:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775776666.940444:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776666.940446:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1775776666.940450:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776666.940452:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cb00. 00000400:00000200:0.0:1775776666.940455:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776666.940458:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776666.940461:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776666.940463:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776666.940466:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776666.940469:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70a80 x1862022710431616/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776691 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940477:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776666.940480:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70a80 x1862022710431616/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776691 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940486:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776666.940487:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776666.940491:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776666.940493:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:0.0:1775776666.940496:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776666.940497:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308221964 00000100:00000001:0.0:1775776666.940499:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308225798 00000100:00000001:0.0:1775776666.940501:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308221964 00000100:00000001:0.0:1775776666.940503:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308225798 00000100:00000001:0.0:1775776666.940504:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776666.940505:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776666.940507:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776666.940509:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776666/real 1775776666] req@ffff8f6202c70a80 x1862022710431616/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776666.940514:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776666.940515:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776666.940516:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776666.940518:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0f68 00000400:00000010:0.0:1775776666.940518:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0f68. 00000100:00000001:0.0:1775776666.940534:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776666.940538:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70a80 x1862022710431616/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776666.940546:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70a80 x1862022710431616/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940556:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776666.940557:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776666.940559:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776666.940560:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776666.940563:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70a80 x1862022710431616/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940570:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776666.940582:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70a80 x1862022710431616/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940588:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776666.940590:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776666.940592:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776666.940593:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776666.940595:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776666.940600:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776666.940602:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776666.940605:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776666.940607:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776666.940611:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70a80 x1862022710431616/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776666.940618:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710431616:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776666.940622:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775776666.940625:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c70700 x1862022710431744/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940631:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776666.940632:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776666.940634:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775776666.940636:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776666.940642:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776666.940645:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710431744:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776666.940648:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775776666.940650:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775776666.940652:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776666.940653:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775776666.940657:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776666.940658:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775776666.940662:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:0.0:1775776666.940664:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f6233888d10. 00000400:00000010:0.0:1775776666.940667:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0f68. 00000100:00000200:0.0:1775776666.940670:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710431744, portal 10 00000100:00000001:0.0:1775776666.940671:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776666.940672:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849005824 : -123823860545792 : ffff8f6202c70700) 00000100:00000040:0.0:1775776666.940678:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c70700 x1862022710431744/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776691 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940683:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775776666.940685:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:0.0:1775776666.940687:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f6203dc0000. 00000100:00000200:0.0:1775776666.940689:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710431744, offset 0 00000400:00000200:0.0:1775776666.940693:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:0.0:1775776666.940696:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776666.940699:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:0.0:1775776666.940702:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775776666.940704:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:0.0:1775776666.940706:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775776666.940708:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:0.0:1775776666.940710:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453925:453925:256:4294967295] 192.168.201.115@tcp LPNI seq info [453850:453850:8:4294967295] 00000400:00000200:0.0:1775776666.940714:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776666.940719:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:0.0:1775776666.940723:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776666.940726:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f610111cb00. 00000800:00000200:0.0:1775776666.940729:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.0:1775776666.940731:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1775776666.940735:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:0.1:1775776666.940737:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1775776666.940741:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:0.0:1775776666.940742:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f610111cb00. 00000400:00000200:0.0:1775776666.940744:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775776666.940747:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775776666.940750:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0000 00000400:00000010:0.0:1775776666.940752:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0000. 00000100:00000001:0.0:1775776666.940754:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775776666.940757:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c70700 x1862022710431744/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776691 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940764:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776666.940766:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c70700 x1862022710431744/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776691 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940772:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776666.940773:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775776666.940776:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776666.940779:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:0.0:1775776666.940782:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776666.940784:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308221964 00000100:00000001:0.0:1775776666.940786:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308225798 00000100:00000001:0.0:1775776666.940787:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308221964 00000100:00000001:0.0:1775776666.940789:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308225798 00000100:00000001:0.0:1775776666.940791:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775776666.940793:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776666.940794:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775776666.940797:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776666/real 1775776666] req@ffff8f6202c70700 x1862022710431744/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775776666.940804:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775776666.940805:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775776666.940807:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f6233888d10. 00000400:00000200:0.0:1775776666.940809:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f6203dc0f68 00000400:00000010:0.0:1775776666.940810:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f6203dc0f68. 00000100:00000001:0.0:1775776666.940812:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775776666.940815:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c70700 x1862022710431744/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775776666.940821:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c70700 x1862022710431744/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940827:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775776666.940828:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776666.940830:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776666.940831:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775776666.940834:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c70700 x1862022710431744/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940839:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775776666.940843:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c70700 x1862022710431744/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940848:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775776666.940850:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776666.940852:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775776666.940853:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776666.940855:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775776666.940857:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775776666.940859:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775776666.940862:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:0.0:1775776666.940864:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775776666.940868:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c70700 x1862022710431744/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775776666.940875:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c70700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710431744:192.168.201.115@tcp:38:kworker.0 00000100:00000001:0.0:1775776666.940878:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776666.940880:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776666.940882:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70a80 x1862022710431616/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940888:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776666.940890:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776666.940891:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:0.0:1775776666.940893:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776666.940895:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776666.940896:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776666.940898:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776666.940899:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34000. 02000000:00000001:0.0:1775776666.940902:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776666.940903:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776666.940904:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70a80. 00000100:00000001:0.0:1775776666.940906:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776666.940907:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776666.940909:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775776666.940912:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c70700 x1862022710431744/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776691 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776666.940918:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775776666.940919:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775776666.940921:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:0.0:1775776666.940923:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775776666.940924:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775776666.940925:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:0.0:1775776666.940926:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775776666.940928:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36c00. 02000000:00000001:0.0:1775776666.940930:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775776666.940931:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775776666.940932:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c70700. 00000100:00000001:0.0:1775776666.940934:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775776666.940936:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776666.940937:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776666.940940:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776666.940942:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776666.940944:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775776666.940945:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776667.067268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776667.067275:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776667.067278:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776667.067281:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776667.067286:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776667.067288:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776667.067290:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776667.067292:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776667.067297:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776667.067299:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776667.067302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776667.067303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776667.067747:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776667.067751:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776667.067754:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776667.963239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776667.963246:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776667.963248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776667.963254:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776667.963258:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776667.963262:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776667.963264:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776667.963266:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776667.963268:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776668.091284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776668.091291:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776668.091296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776668.091298:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776668.091302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776668.091303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776668.091371:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776668.091375:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776668.091378:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776668.987626:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776668.987632:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776668.987633:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776668.987638:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776668.987640:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776668.987644:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776668.987646:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776668.987648:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776668.987650:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776669.115323:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776669.115330:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776669.115349:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776669.115351:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776669.115359:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776669.115360:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776669.116011:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776669.116016:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776669.116020:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1775776670.011355:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776670.011371:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776670.011373:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776670.011381:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776670.011385:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776670.011389:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776670.011393:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776670.011395:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776670.011398:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776670.139247:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776670.139256:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776670.139262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776670.139263:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776670.139266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776670.139266:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776670.139537:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776670.139540:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776670.139543:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776670.843307:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308225798, at 4308225856 00000100:00000040:1.0:1775776670.843321:0:214731:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308225798, at 4308225856 00000100:00000010:0.0:1775776670.843327:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c3c00. 00000100:00000010:1.0:1775776670.843340:0:214731:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62264c36c0. 00000100:00000001:0.0:1775776671.035342:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1775776671.035351:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775776671.035353:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776671.035361:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:0.0:1775776671.035365:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775776671.035369:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776671.035373:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775776671.035376:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776671.035379:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776671.163257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776671.163264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776671.163269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776671.163270:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776671.163274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776671.163276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776671.163372:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776671.163376:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776671.163379:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776672.059299:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776672.059313:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775776672.059316:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776672.059321:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776672.059327:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776672.059336:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13554 00000400:00000200:0.0:1775776672.059348:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776672.059355:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776672.059359:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776672.059363:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776672.059366:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776672.059368:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776672.059371:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776672.059376:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776672.059378:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776672.059381:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776672.059384:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776672.059387:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776672.059389:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776672.059392:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776672.059394:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776672.059396:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776672.059398:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776672.059401:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776672.059404:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776672.059406:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776672.059410:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776672.059415:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776672.059461:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776672.059464:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776672.059466:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776672.059478:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061c00. 00000020:00000040:0.0:1775776672.059483:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776672.059488:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776672.059490:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776672.059494:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776672.059497:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776672.059503:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776672.059513:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e7800. 00000100:00000001:0.0:1775776672.059522:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776672.059535:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061c00 x1862022710431872/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776672.059546:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776672.059552:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776672.059557:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076a44879] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061c00 x1862022710431872/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776672.059585:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776672.059589:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776672.059593:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776672.059594:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776672.059596:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776672.059599:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776672.059602:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13554 00000400:00000200:0.0:1775776672.059606:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776672.059610:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776672.059613:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776672.059617:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776672.059620:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776672.059622:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776672.059625:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776672.059627:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776672.059629:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776672.059631:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776672.059633:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775776672.059635:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775776672.059638:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775776672.059642:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776672.059644:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776672.059644:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1775776672.059648:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776672.059652:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000100:00000001:1.0:1775776672.059653:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1775776672.059657:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00000001:1.0:1775776672.059658:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1775776672.059665:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776672.059668:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776672.059674:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1775776672.059675:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061c00 x1862022710431872/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1775776672.059681:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776672.059690:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776672.059696:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:1.0:1775776672.059703:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000008:00100000:0.0:1775776672.059705:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1775776672.059706:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775776672.059714:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776672.059715:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1775776672.059718:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776672.059721:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:0.0:1775776672.059723:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e061880. 00000020:00000040:0.0:1775776672.059729:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776672.059729:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776672.059739:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776672.059742:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776672.059744:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710431872:192.168.201.115@tcp:250:kworker.0 00000100:00000001:0.0:1775776672.059746:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776672.059748:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776672.059754:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776672.059754:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000010:0.0:1775776672.059762:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e5800. 02000000:00000001:1.0:1775776672.059763:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1775776672.059767:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776672.059768:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776672.059779:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:0.0:1775776672.059785:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e061880 x1862022710432000/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1775776672.059795:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4f400. 02000000:00000001:1.0:1775776672.059804:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776672.059813:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776672.059818:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776672.059821:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:0.0:1775776672.059827:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005afd1f27] to pc [ptlrpcd_rcv+-1] req@ffff8f622e061880 x1862022710432000/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1775776672.059831:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000001:0.0:1775776672.059843:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776672.059844:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00080000:0.0:1775776672.059847:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776672.059852:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000200:1.0:1775776672.059853:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710431872, portal 25 00000100:00000001:0.0:1775776672.059854:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776672.059856:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1775776672.059859:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775776672.059863:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:1.0:1775776672.059863:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574560256 : -123823134991360 : ffff8f622e061c00) 00000100:00080000:0.0:1775776672.059869:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13554 00000100:00000040:1.0:1775776672.059876:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061c00 x1862022710431872/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1775776672.059880:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776672.059891:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776672.059898:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776672.059907:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776672.059908:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1775776672.059913:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776672.059920:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000040:1.0:1775776672.059922:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000001:0.0:1775776672.059928:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000010:1.0:1775776672.059931:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000020:00000001:0.0:1775776672.059933:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776672.059937:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000100:00000200:1.0:1775776672.059939:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710431872, offset 0 00000020:00000001:0.0:1775776672.059940:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776672.059947:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775776672.059950:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775776672.059953:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000400:00000200:1.0:1775776672.059958:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00000040:0.0:1775776672.059963:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776672.059967:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776672.059974:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776672.059975:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:0.0:1775776672.059983:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776672.059989:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000400:00000200:1.0:1775776672.059990:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00080000:0.0:1775776672.060000:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000400:00000200:1.0:1775776672.060002:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1775776672.060004:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776672.060009:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776672.060013:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776672.060016:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776672.060018:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776672.060020:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1775776672.060023:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000008:00000001:0.0:1775776672.060027:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776672.060033:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1775776672.060034:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776672.060039:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000010:0.0:1775776672.060041:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060700. 00000400:00000200:1.0:1775776672.060049:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453926:453926:256:4294967295] 192.168.201.115@tcp LPNI seq info [453851:453851:8:4294967295] 00000400:00000200:1.0:1775776672.060054:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000020:00000040:0.0:1775776672.060061:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776672.060071:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1775776672.060071:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 02000000:00000001:0.0:1775776672.060076:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776672.060081:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776672.060084:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:1.0:1775776672.060087:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 02000000:00000001:0.0:1775776672.060088:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776672.060095:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4800. 00000800:00000010:1.0:1775776672.060096:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000100:00000001:0.0:1775776672.060103:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776672.060111:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00100000:0.0:1775776672.060116:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060700 x1862022710432128/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1775776672.060118:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775776672.060167:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776672.060178:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000100:00000001:0.0:1775776672.060179:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776672.060183:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776672.060190:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000225c8695] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060700 x1862022710432128/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776672.060204:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776672.060207:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1775776672.060217:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000040:0.0:1775776672.060218:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000010:1.0:1775776672.060224:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000100:00000040:0.0:1775776672.060225:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776672.060236:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1775776672.060239:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:0.0:1775776672.060246:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13564) 00000100:00000001:2.0:1775776672.060246:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000400:00000200:1.0:1775776672.060253:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1775776672.060256:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:1.0:1775776672.060260:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000100:00000001:2.0:1775776672.060260:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776672.060265:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00100000:2.0:1775776672.060266:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776672.060271:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776672.060272:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1775776672.060277:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1775776672.060279:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061c00 x1862022710431872/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776672.060280:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776672.060284:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776672.060291:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776672.060295:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776672.060310:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061c00 x1862022710431872/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776697 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.060317:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776672.060320:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776672.060327:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776672.060332:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776672.060335:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308230919 00000100:00000010:1.0:1775776672.060340:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2c60. 00000100:00000001:1.0:1775776672.060347:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776672.060353:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308230919 00000100:00000010:1.0:1775776672.060356:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2ae0. 00000100:00000001:1.0:1775776672.060359:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776672.060362:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776672.060364:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776672.060367:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776672.060370:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776672/real 1775776672] req@ffff8f622e061c00 x1862022710431872/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776672.060379:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776672.060381:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776672.060383:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776672.060385:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776672.060387:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776672.060389:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776672.060393:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061c00 x1862022710431872/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776672.060402:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061c00 x1862022710431872/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.060409:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776672.060410:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776672.060413:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776672.060415:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776672.060418:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061c00 x1862022710431872/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.060450:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776672.060455:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061c00 x1862022710431872/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.060462:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776672.060464:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776672.060466:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776672.060467:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776672.060470:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776672.060472:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776672.060475:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776672.060479:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776672.060481:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776672.060485:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061c00 x1862022710431872/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776672.060493:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710431872:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776672.060497:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776672.060499:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776672.060502:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061c00 x1862022710431872/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.060508:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776672.060511:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776672.060513:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4f400. 02000000:00000001:1.0:1775776672.060516:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776672.060519:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776672.060520:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776672.060522:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776672.060524:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e7800. 02000000:00000001:1.0:1775776672.060528:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776672.060530:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776672.060532:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061c00. 00000100:00000001:1.0:1775776672.060536:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776672.060537:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776672.060539:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776672.060546:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776672.060547:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776672.060549:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776672.060551:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776672.060552:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776672.060556:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e061880 x1862022710432000/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.060567:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776672.060568:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776672.060570:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776672.060572:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776672.060575:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776672.060577:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e061880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710432000:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776672.060580:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776672.060582:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776672.060584:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776672.060585:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776672.060588:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4f400. 02000000:00000001:1.0:1775776672.060589:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776672.060593:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776672.060596:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776672.060600:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776672.060602:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710432000, portal 10 00000100:00000001:1.0:1775776672.060604:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776672.060605:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574559360 : -123823134992256 : ffff8f622e061880) 00000100:00000040:1.0:1775776672.060611:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e061880 x1862022710432000/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.060616:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776672.060618:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776672.060620:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776672.060622:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710432000, offset 0 00000400:00000200:1.0:1775776672.060625:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776672.060629:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776672.060633:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776672.060635:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776672.060637:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776672.060640:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776672.060642:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776672.060644:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453927:453927:256:4294967295] 192.168.201.115@tcp LPNI seq info [453852:453852:8:4294967295] 00000400:00000200:1.0:1775776672.060648:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776672.060653:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776672.060657:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776672.060659:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776672.060663:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776672.060665:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775776672.060669:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776672.060672:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1775776672.060676:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776672.060677:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776672.060680:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776672.060683:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776672.060686:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776672.060687:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776672.060689:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776672.060692:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e061880 x1862022710432000/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.060698:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776672.060700:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e061880 x1862022710432000/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776697 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.060705:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776672.060707:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776672.060709:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776672.060713:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776672.060715:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776672.060717:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308227084 00000100:00000001:1.0:1775776672.060719:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308230919 00000100:00000001:1.0:1775776672.060721:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308227084 00000100:00000001:1.0:1775776672.060723:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308230919 00000100:00000001:1.0:1775776672.060724:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776672.060727:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776672.060728:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776672.060731:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776672/real 1775776672] req@ffff8f622e061880 x1862022710432000/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776672.060737:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776672.060738:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776672.060740:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776672.060742:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776672.060743:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776672.060745:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776672.060748:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e061880 x1862022710432000/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776672.060756:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e061880 x1862022710432000/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.060763:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776672.060764:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776672.060766:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776672.060767:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776672.060770:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e061880 x1862022710432000/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.060777:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776672.060779:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e061880 x1862022710432000/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.060784:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776672.060785:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776672.060786:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776672.060787:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776672.060788:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776672.060792:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776672.060793:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776672.060796:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776672.060797:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776672.060799:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e061880 x1862022710432000/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776672.060804:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e061880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710432000:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776672.060806:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776672.060808:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e060700 x1862022710432128/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.060811:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776672.060812:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776672.060814:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776672.060815:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776672.060816:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776672.060817:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e060700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710432128:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776672.060819:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776672.060821:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776672.060822:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776672.060823:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776672.060825:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776672.060826:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776672.060829:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776672.060830:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776672.060832:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776672.060834:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710432128, portal 10 00000100:00000001:1.0:1775776672.060835:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776672.060836:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574554880 : -123823134996736 : ffff8f622e060700) 00000100:00000040:1.0:1775776672.060840:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e060700 x1862022710432128/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.060844:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776672.060846:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776672.060847:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776672.060849:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710432128, offset 0 00000400:00000200:1.0:1775776672.060851:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776672.060854:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776672.060856:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776672.060858:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776672.060860:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776672.060861:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776672.060863:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776672.060865:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453928:453928:256:4294967295] 192.168.201.115@tcp LPNI seq info [453853:453853:8:4294967295] 00000400:00000200:1.0:1775776672.060868:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776672.060872:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776672.060874:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776672.060876:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776672.060879:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776672.060881:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1775776672.060883:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776672.060885:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1775776672.060888:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776672.060889:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776672.060891:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776672.060893:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776672.060895:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776672.060896:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776672.060897:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776672.060899:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e060700 x1862022710432128/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.060910:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776672.060912:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e060700 x1862022710432128/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776697 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.060916:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776672.060917:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776672.060919:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776672.060921:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776672.060923:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776672.060924:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308227084 00000100:00000001:1.0:1775776672.060926:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308230919 00000100:00000001:1.0:1775776672.060927:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308227084 00000100:00000001:1.0:1775776672.060928:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308230919 00000100:00000001:1.0:1775776672.060930:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776672.060931:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776672.060932:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776672.060934:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776672/real 1775776672] req@ffff8f622e060700 x1862022710432128/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776672.060938:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776672.060938:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776672.060939:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776672.060940:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776672.060941:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776672.060942:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776672.060944:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e060700 x1862022710432128/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776672.060948:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e060700 x1862022710432128/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.060952:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776672.060953:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776672.060954:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776672.060954:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776672.060956:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e060700 x1862022710432128/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.060959:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776672.060962:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e060700 x1862022710432128/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.060966:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776672.060968:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776672.060968:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776672.060969:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776672.060970:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776672.060971:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776672.060972:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776672.060974:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776672.060975:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776672.060977:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e060700 x1862022710432128/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776672.060981:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e060700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710432128:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776672.060983:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776672.060984:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776672.060986:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e061880 x1862022710432000/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.060989:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776672.060990:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776672.060992:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4f400. 02000000:00000001:1.0:1775776672.060993:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776672.060994:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776672.060995:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776672.060996:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776672.060997:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e5800. 02000000:00000001:1.0:1775776672.060998:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776672.060999:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776672.061000:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e061880. 00000100:00000001:1.0:1775776672.061002:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776672.061003:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776672.061004:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776672.061006:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e060700 x1862022710432128/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776697 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776672.061009:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776672.061010:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776672.061011:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776672.061012:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776672.061012:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776672.061013:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776672.061014:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776672.061014:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e4800. 02000000:00000001:1.0:1775776672.061016:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776672.061016:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776672.061017:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e060700. 00000100:00000001:1.0:1775776672.061018:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776672.061019:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776672.061020:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776672.061022:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776672.061023:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776672.061024:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776672.061024:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776672.187271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776672.187277:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776672.187282:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776672.187285:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776672.187287:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776672.187290:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776672.187292:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776672.187295:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776672.187296:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776672.187299:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776672.187302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776672.187303:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776672.187304:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776672.187305:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776672.187307:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776672.187308:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776672.187309:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776672.187311:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776672.187371:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776672.187375:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776672.187378:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776673.083958:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776673.083965:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776673.083967:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776673.083972:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776673.083975:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776673.083978:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776673.083980:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776673.083983:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776673.083985:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776673.211349:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776673.211355:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776673.211361:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776673.211363:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776673.211367:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776673.211368:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776673.211452:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776673.211455:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776673.211458:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776674.107252:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776674.107258:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776674.107260:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776674.107266:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776674.107270:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776674.107274:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776674.107276:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776674.107279:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776674.107281:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776674.235528:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776674.235533:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776674.235539:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776674.235540:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776674.235543:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776674.235544:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776674.235612:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776674.235616:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776674.235619:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776675.131413:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776675.131421:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776675.131423:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776675.131429:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776675.131433:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776675.131436:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776675.131438:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776675.131441:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776675.131444:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776675.259269:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776675.259275:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776675.259280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776675.259281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776675.259286:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776675.259287:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776675.259434:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776675.259438:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776675.259441:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776675.963243:0:214731:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308230919, at 4308230976 00000100:00000010:1.0:1775776675.963257:0:214731:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ae0. 00000100:00000040:0.0:1775776675.963341:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308230919, at 4308230976 00000100:00000010:0.0:1775776675.963348:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2c60. 00000100:00000001:2.0:1775776676.155254:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776676.155260:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776676.155262:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776676.155267:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776676.155270:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776676.155274:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776676.155276:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776676.155279:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776676.155281:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776676.283255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776676.283261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776676.283266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776676.283267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776676.283271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776676.283272:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776676.283327:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776676.283331:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776676.283334:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775776677.179231:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776677.179232:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776677.179237:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1775776677.179239:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776677.179240:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776677.179243:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1775776677.179245:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:0.0:1775776677.179246:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776677.179249:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776677.179254:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13559 00000100:00000001:2.0:1775776677.179261:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1775776677.179262:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:2.0:1775776677.179267:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1775776677.179268:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776677.179272:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1775776677.179272:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1775776677.179276:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776677.179276:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775776677.179280:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1775776677.179282:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776677.179283:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776677.179287:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776677.179290:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776677.179291:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:0.0:1775776677.179293:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776677.179294:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:0.0:1775776677.179296:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:0.0:1775776677.179298:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:0.0:1775776677.179300:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776677.179301:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776677.179302:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776677.179304:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776677.179306:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:0.0:1775776677.179308:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776677.179309:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776677.179311:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776677.179314:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776677.179315:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776677.179317:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775776677.179319:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776677.179326:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e063100. 00000020:00000040:0.0:1775776677.179328:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:0.0:1775776677.179331:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776677.179332:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776677.179333:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776677.179336:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776677.179339:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776677.179356:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e6400. 00000100:00000001:0.0:1775776677.179360:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776677.179368:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e063100 x1862022710432256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776677.179375:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776677.179378:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776677.179381:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033f6db7f] to pc [ptlrpcd_rcv+-1] req@ffff8f622e063100 x1862022710432256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776677.179399:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776677.179401:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776677.179403:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775776677.179404:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776677.179406:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776677.179407:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776677.179409:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13559 00000400:00000200:0.0:1775776677.179412:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776677.179414:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776677.179415:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776677.179417:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776677.179418:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776677.179419:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776677.179421:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776677.179422:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776677.179423:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:0.0:1775776677.179423:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776677.179425:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:0.0:1775776677.179426:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:0.0:1775776677.179427:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:0.0:1775776677.179428:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776677.179429:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776677.179430:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776677.179432:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776677.179433:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:0.0:1775776677.179435:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776677.179436:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776677.179437:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776677.179441:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776677.179442:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776677.179443:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776677.179446:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776677.179448:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776677.179449:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776677.179452:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060e00. 00000020:00000040:0.0:1775776677.179453:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776677.179455:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776677.179456:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776677.179457:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776677.179458:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776677.179459:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776677.179461:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4c00. 00000100:00000001:0.0:1775776677.179463:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776677.179466:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060e00 x1862022710432384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776677.179470:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776677.179471:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776677.179473:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004a3bddaa] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060e00 x1862022710432384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776677.179477:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775776677.179479:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775776677.179484:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1775776677.179485:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775776677.179485:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775776677.179487:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775776677.179488:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13559 00000400:00000200:0.0:1775776677.179490:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:0.0:1775776677.179491:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:0.0:1775776677.179493:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775776677.179495:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776677.179496:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776677.179497:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776677.179498:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:0.0:1775776677.179499:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775776677.179500:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:0.0:1775776677.179501:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775776677.179502:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:0.0:1775776677.179503:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:0.0:1775776677.179505:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:0.0:1775776677.179507:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776677.179508:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775776677.179509:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:0.0:1775776677.179510:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:0.0:1775776677.179511:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:0.0:1775776677.179513:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:0.0:1775776677.179514:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775776677.179515:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775776677.179516:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775776677.179517:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776677.179518:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775776677.179519:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1775776677.179520:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776677.179521:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775776677.179524:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f622e060700. 00000020:00000040:0.0:1775776677.179525:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:0.0:1775776677.179526:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775776677.179527:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775776677.179528:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775776677.179528:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775776677.179529:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775776677.179531:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f622e0e4400. 00000100:00000001:0.0:1775776677.179533:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775776677.179535:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f622e060700 x1862022710432512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775776677.179538:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775776677.179539:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776677.179541:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000225c8695] to pc [ptlrpcd_rcv+-1] req@ffff8f622e060700 x1862022710432512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775776677.179546:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775776677.179548:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776677.179552:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776677.179554:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1775776677.179556:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1775776677.179559:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13569) 00000100:00000001:1.0:1775776677.179736:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776677.179740:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776677.179742:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776677.179749:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e063100 x1862022710432256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.179757:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776677.179759:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776677.179762:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776677.179765:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776677.179768:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776677.179771:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e063100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710432256:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776677.179775:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776677.179779:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776677.179782:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776677.179784:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776677.179789:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776677.179792:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776677.179797:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776677.179801:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776677.179807:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776677.179814:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710432256, portal 25 00000100:00000001:1.0:1775776677.179816:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776677.179818:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574565632 : -123823134985984 : ffff8f622e063100) 00000100:00000040:1.0:1775776677.179823:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e063100 x1862022710432256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776702 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.179830:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776677.179833:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776677.179836:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776677.179838:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710432256, offset 0 00000400:00000200:1.0:1775776677.179845:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776677.179852:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776677.179858:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776677.179863:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776677.179867:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776677.179870:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776677.179873:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776677.179878:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453929:453929:256:4294967295] 192.168.201.115@tcp LPNI seq info [453854:453854:8:4294967295] 00000400:00000200:1.0:1775776677.179882:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776677.179891:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776677.179896:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776677.179902:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776677.179909:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776677.179913:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775776677.179918:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776677.179921:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1775776677.179925:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776677.179928:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776677.179933:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776677.179938:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776677.179942:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776677.179945:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776677.179950:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776677.179954:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e063100 x1862022710432256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776702 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.179962:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776677.179965:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e063100 x1862022710432256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776702 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.179972:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776677.179974:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776677.179979:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776677.179983:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776677.179985:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308236038 00000100:00000010:1.0:1775776677.179990:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2ae0. 00000100:00000001:1.0:1775776677.179997:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776677.180003:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308236038 00000100:00000010:1.0:1775776677.180005:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2cc0. 00000100:00000001:1.0:1775776677.180009:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776677.180012:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776677.180015:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776677.180017:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776677.180021:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776677/real 1775776677] req@ffff8f622e063100 x1862022710432256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776677.180031:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776677.180032:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776677.180034:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776677.180037:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776677.180038:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776677.180041:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776677.180044:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e063100 x1862022710432256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776677.180052:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e063100 x1862022710432256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.180059:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776677.180060:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776677.180063:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776677.180064:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776677.180068:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e063100 x1862022710432256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.180075:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776677.180078:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e063100 x1862022710432256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.180085:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776677.180087:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776677.180089:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776677.180090:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776677.180092:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776677.180094:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776677.180097:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776677.180101:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776677.180104:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776677.180108:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e063100 x1862022710432256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776677.180115:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e063100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710432256:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776677.180120:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776677.180123:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e060e00 x1862022710432384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.180129:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776677.180131:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776677.180156:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776677.180158:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776677.180160:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776677.180163:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e060e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710432384:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776677.180167:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776677.180169:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776677.180170:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776677.180172:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776677.180175:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776677.180177:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776677.180181:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776677.180183:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776677.180186:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776677.180189:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710432384, portal 10 00000100:00000001:1.0:1775776677.180191:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776677.180192:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574556672 : -123823134994944 : ffff8f622e060e00) 00000100:00000040:1.0:1775776677.180198:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e060e00 x1862022710432384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776702 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.180203:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776677.180206:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776677.180207:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776677.180210:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710432384, offset 0 00000400:00000200:1.0:1775776677.180213:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776677.180217:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776677.180220:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776677.180223:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776677.180225:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776677.180228:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776677.180230:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776677.180232:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453930:453930:256:4294967295] 192.168.201.115@tcp LPNI seq info [453855:453855:8:4294967295] 00000400:00000200:1.0:1775776677.180237:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776677.180243:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776677.180248:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776677.180251:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776677.180254:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776677.180256:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775776677.180260:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776677.180262:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1775776677.180266:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776677.180268:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776677.180270:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776677.180273:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776677.180276:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776677.180277:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776677.180279:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776677.180283:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e060e00 x1862022710432384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776702 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.180290:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776677.180293:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e060e00 x1862022710432384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776702 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.180299:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776677.180300:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776677.180304:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776677.180308:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776677.180311:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776677.180313:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308232204 00000100:00000001:1.0:1775776677.180315:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308236038 00000100:00000001:1.0:1775776677.180317:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308232204 00000100:00000001:1.0:1775776677.180319:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308236038 00000100:00000001:1.0:1775776677.180321:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776677.180323:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776677.180324:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776677.180328:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776677/real 1775776677] req@ffff8f622e060e00 x1862022710432384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776677.180335:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776677.180337:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776677.180338:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776677.180354:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776677.180356:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776677.180359:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776677.180361:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e060e00 x1862022710432384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776677.180370:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e060e00 x1862022710432384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.180376:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776677.180378:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776677.180380:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776677.180381:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776677.180384:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e060e00 x1862022710432384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.180390:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776677.180394:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e060e00 x1862022710432384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.180401:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776677.180403:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776677.180405:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776677.180406:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776677.180408:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776677.180412:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776677.180414:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776677.180417:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776677.180419:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776677.180423:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e060e00 x1862022710432384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776677.180430:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e060e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710432384:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776677.180434:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776677.180436:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f622e060700 x1862022710432512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.180443:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776677.180444:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776677.180446:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776677.180448:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776677.180449:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776677.180452:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f622e060700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710432512:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776677.180455:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776677.180457:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776677.180458:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776677.180460:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776677.180463:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4c000. 02000000:00000001:1.0:1775776677.180465:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776677.180468:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776677.180471:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776677.180474:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776677.180477:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710432512, portal 10 00000100:00000001:1.0:1775776677.180478:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776677.180479:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250574554880 : -123823134996736 : ffff8f622e060700) 00000100:00000040:1.0:1775776677.180484:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f622e060700 x1862022710432512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776702 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.180491:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776677.180493:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776677.180495:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776677.180497:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710432512, offset 0 00000400:00000200:1.0:1775776677.180501:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776677.180504:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776677.180507:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776677.180510:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776677.180512:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776677.180515:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776677.180517:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776677.180519:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453931:453931:256:4294967295] 192.168.201.115@tcp LPNI seq info [453856:453856:8:4294967295] 00000400:00000200:1.0:1775776677.180523:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776677.180529:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776677.180533:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776677.180536:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776677.180540:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776677.180543:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1775776677.180546:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776677.180549:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1775776677.180553:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776677.180554:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776677.180557:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776677.180561:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776677.180563:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776677.180565:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776677.180567:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776677.180570:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f622e060700 x1862022710432512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776702 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.180578:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776677.180581:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f622e060700 x1862022710432512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776702 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.180587:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776677.180589:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776677.180591:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776677.180593:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776677.180596:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776677.180598:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308232204 00000100:00000001:1.0:1775776677.180600:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308236038 00000100:00000001:1.0:1775776677.180602:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308232204 00000100:00000001:1.0:1775776677.180605:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308236038 00000100:00000001:1.0:1775776677.180606:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776677.180608:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776677.180610:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776677.180614:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776677/real 1775776677] req@ffff8f622e060700 x1862022710432512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776677.180621:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776677.180622:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776677.180624:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776677.180626:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776677.180628:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776677.180630:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776677.180633:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f622e060700 x1862022710432512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776677.181807:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f622e060700 x1862022710432512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.181819:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776677.181821:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776677.181823:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776677.181825:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776677.181828:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f622e060700 x1862022710432512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.181835:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776677.181839:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f622e060700 x1862022710432512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.181847:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776677.181850:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776677.181851:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776677.181853:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776677.181856:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776677.181858:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776677.181861:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776677.181866:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776677.181868:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776677.181876:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f622e060700 x1862022710432512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776677.181885:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f622e060700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710432512:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776677.181889:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776677.181892:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776677.181895:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e063100 x1862022710432256/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.181902:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776677.181905:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776677.181908:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776677.181913:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776677.181915:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776677.181917:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776677.181919:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776677.181921:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e6400. 02000000:00000001:1.0:1775776677.181925:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776677.181927:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776677.181928:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e063100. 00000100:00000001:1.0:1775776677.181932:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776677.181933:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776677.181935:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776677.181938:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e060e00 x1862022710432384/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.181944:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776677.181946:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776677.181947:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776677.181950:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776677.181951:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776677.181952:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776677.181955:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776677.181956:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e4c00. 02000000:00000001:1.0:1775776677.181959:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776677.181959:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776677.181960:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e060e00. 00000100:00000001:1.0:1775776677.181962:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776677.181963:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776677.181965:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776677.181968:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f622e060700 x1862022710432512/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776702 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776677.181975:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776677.181976:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776677.181978:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4c000. 02000000:00000001:1.0:1775776677.181985:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776677.181987:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776677.181988:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776677.181991:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776677.181992:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f622e0e4400. 02000000:00000001:1.0:1775776677.181995:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776677.181996:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776677.181997:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f622e060700. 00000100:00000001:1.0:1775776677.181999:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776677.182000:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776677.182002:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776677.182009:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776677.182010:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776677.182013:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776677.182014:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776677.308235:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776677.308242:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776677.308245:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776677.308248:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776677.308252:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776677.308255:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776677.308257:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776677.308259:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776677.308262:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776677.308264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776677.308268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776677.308269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776677.308390:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776677.308393:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776677.308395:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776678.203297:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776678.203307:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776678.203309:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776678.203316:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776678.203321:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776678.203340:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776678.203343:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776678.203346:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776678.203349:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776678.331291:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776678.331297:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776678.331302:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776678.331303:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776678.331306:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776678.331307:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776678.331381:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776678.331385:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776678.331388:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776679.227776:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776679.227792:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776679.227794:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776679.227801:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776679.227805:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776679.227810:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776679.227813:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776679.227815:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776679.227819:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776679.355296:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776679.355326:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776679.355335:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776679.355336:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776679.355341:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776679.355343:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776679.355405:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776679.355410:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776679.355416:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776680.251238:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776680.251245:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776680.251247:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776680.251252:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776680.251255:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776680.251259:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776680.251262:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776680.251264:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776680.251267:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776680.379255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776680.379260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776680.379264:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776680.379264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776680.379267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776680.379268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776680.379335:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776680.379339:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776680.379342:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776681.083270:0:214731:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308236038, at 4308236096 00000100:00000040:0.0:1775776681.083282:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308236038, at 4308236096 00000100:00000010:1.0:1775776681.083303:0:214731:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2cc0. 00000100:00000010:0.0:1775776681.083307:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2ae0. 00000800:00000200:3.0:1775776681.087534:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6220175d80] -> 12345-192.168.201.15@tcp (6) 00000800:00000200:3.0:1775776681.087542:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.15@tcp ip 192.168.201.15:988 00000800:00000200:3.0:1775776681.087547:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8f620b3fb300 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1775776681.087925:0:2404:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1775776681.087931:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776681.087955:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1775776681.087960:0:2404:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1775776681.087965:0:2404:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776681.275221:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776681.275227:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776681.275228:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776681.275233:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776681.275237:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776681.275240:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776681.275243:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776681.275245:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776681.275248:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776681.403388:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776681.403392:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776681.403397:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776681.403401:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776681.403403:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776681.403406:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776681.403409:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776681.403411:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776681.403414:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776681.403416:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776681.403419:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776681.403421:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776681.403422:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776681.403423:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776681.403425:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776681.403426:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776681.403427:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776681.403429:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776681.403532:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776681.403538:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776681.403542:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776682.299275:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776682.299282:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776682.299284:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776682.299290:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776682.299293:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776682.299297:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776682.299300:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776682.299302:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776682.299305:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776682.301851:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776682.301859:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776682.301861:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776682.301865:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776682.301869:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776682.301876:0:141703:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13564 00000400:00000200:2.0:1775776682.301885:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776682.301892:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776682.301896:0:141703:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776682.301900:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776682.301903:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776682.301906:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776682.301908:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776682.301911:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776682.301913:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776682.301915:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776682.301918:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776682.301920:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776682.301922:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776682.301925:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776682.301927:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776682.301928:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776682.301930:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776682.301934:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776682.301937:0:141703:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776682.301940:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776682.301942:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776682.301946:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776682.301949:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776682.301951:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776682.301953:0:141703:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776682.301964:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344aa00. 00000020:00000040:2.0:1775776682.301968:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776682.301971:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776682.301972:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776682.301975:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776682.301978:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776682.301982:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776682.301992:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e800. 00000100:00000001:2.0:1775776682.301998:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776682.302007:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344aa00 x1862022710432640/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776682.302018:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776682.302022:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776682.302027:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70bc0bf] to pc [ptlrpcd_rcv+-1] req@ffff8f623344aa00 x1862022710432640/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776682.302058:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776682.302061:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776682.302064:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776682.302065:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776682.302067:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776682.302069:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776682.302072:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13564 00000400:00000200:2.0:1775776682.302076:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776682.302079:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776682.302081:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776682.302084:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776682.302086:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776682.302088:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776682.302090:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776682.302092:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776682.302093:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776682.302095:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776682.302096:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776682.302098:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776682.302100:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776682.302102:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776682.302105:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776682.302106:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776682.302108:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776682.302111:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776682.302113:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776682.302115:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776682.302117:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776682.302121:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776682.302123:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776682.302125:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776682.302129:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776682.302132:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776682.302157:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776682.302161:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449500. 00000020:00000040:2.0:1775776682.302164:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776682.302166:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776682.302168:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776682.302169:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776682.302170:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776682.302172:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776682.302176:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775776682.302179:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776682.302184:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449500 x1862022710432768/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776682.302192:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776682.302194:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776682.302198:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bf7d100] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449500 x1862022710432768/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776682.302206:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776682.302208:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776682.302211:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776682.302213:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776682.302214:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776682.302217:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776682.302220:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13564 00000400:00000200:2.0:1775776682.302224:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776682.302227:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776682.302229:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776682.302232:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776682.302234:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776682.302236:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776682.302238:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776682.302240:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776682.302241:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776682.302242:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776682.302244:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776682.302246:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776682.302248:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776682.302250:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776682.302264:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776682.302266:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776682.302268:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776682.302270:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776682.302273:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776682.302275:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776682.302277:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776682.302279:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776682.302281:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776682.302283:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776682.302285:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776682.302287:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776682.302288:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776682.302292:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344ad80. 00000020:00000040:2.0:1775776682.302295:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:2.0:1775776682.302297:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776682.302298:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776682.302300:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776682.302301:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776682.302302:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776682.302306:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 00000100:00000001:2.0:1775776682.302309:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776682.302314:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344ad80 x1862022710432896/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776682.302321:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776682.302323:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776682.302327:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dd0683d] to pc [ptlrpcd_rcv+-1] req@ffff8f623344ad80 x1862022710432896/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776682.302333:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776682.302336:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776682.302339:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776682.302342:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776682.302345:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1775776682.302350:0:141703:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13575) 00000100:00000001:1.0:1775776682.302576:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776682.302580:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776682.302582:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776682.302588:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344aa00 x1862022710432640/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.302596:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776682.302598:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776682.302601:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776682.302604:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776682.302607:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776682.302610:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710432640:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776682.302614:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776682.302619:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776682.302622:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776682.302624:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776682.302629:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776682.302632:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776682.302637:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776682.302641:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776682.302648:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776682.302655:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710432640, portal 25 00000100:00000001:1.0:1775776682.302658:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776682.302659:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662545920 : -123823047005696 : ffff8f623344aa00) 00000100:00000040:1.0:1775776682.302665:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344aa00 x1862022710432640/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776707 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.302673:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776682.302675:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776682.302677:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776682.302680:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710432640, offset 0 00000400:00000200:1.0:1775776682.302686:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776682.302693:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776682.302699:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776682.302703:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776682.302707:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776682.302711:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776682.302713:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776682.302717:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453932:453932:256:4294967295] 192.168.201.115@tcp LPNI seq info [453857:453857:8:4294967295] 00000400:00000200:1.0:1775776682.302722:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776682.302731:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776682.302748:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776682.302754:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776682.302760:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776682.302764:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775776682.302769:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776682.302772:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775776682.302776:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776682.302779:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776682.302785:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776682.302790:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776682.302795:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776682.302797:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776682.302803:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776682.302807:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344aa00 x1862022710432640/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776707 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.302816:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776682.302820:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344aa00 x1862022710432640/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776707 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.302827:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776682.302829:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776682.302834:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776682.302837:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776682.302840:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308241161 00000100:00000010:1.0:1775776682.302845:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e21e0. 00000100:00000001:1.0:1775776682.302852:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776682.302857:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308241161 00000100:00000010:1.0:1775776682.302860:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e29c0. 00000100:00000001:1.0:1775776682.302863:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776682.302867:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776682.302869:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776682.302871:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776682.302875:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776682/real 1775776682] req@ffff8f623344aa00 x1862022710432640/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776682.302885:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776682.302887:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776682.302889:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776682.302892:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776682.302893:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776682.302896:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776682.302900:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344aa00 x1862022710432640/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776682.302907:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344aa00 x1862022710432640/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.302914:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776682.302915:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776682.302918:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776682.302919:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776682.302923:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344aa00 x1862022710432640/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.302929:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776682.302933:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344aa00 x1862022710432640/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.302940:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776682.302942:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776682.302943:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776682.302945:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776682.302947:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776682.302949:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776682.302951:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776682.302954:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776682.302956:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776682.302960:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344aa00 x1862022710432640/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776682.302968:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710432640:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776682.302972:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776682.302975:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449500 x1862022710432768/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.302987:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776682.302988:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776682.302990:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776682.302992:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776682.302994:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776682.302997:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710432768:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776682.303000:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776682.303001:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776682.303003:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776682.303005:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776682.303008:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776682.303009:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776682.303013:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776682.303015:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776682.303019:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776682.303021:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710432768, portal 10 00000100:00000001:1.0:1775776682.303023:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776682.303024:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662540544 : -123823047011072 : ffff8f6233449500) 00000100:00000040:1.0:1775776682.303030:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449500 x1862022710432768/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776707 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.303036:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776682.303038:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776682.303041:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776682.303043:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710432768, offset 0 00000400:00000200:1.0:1775776682.303046:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776682.303050:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776682.303053:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776682.303056:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776682.303058:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776682.303060:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776682.303063:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776682.303066:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453933:453933:256:4294967295] 192.168.201.115@tcp LPNI seq info [453858:453858:8:4294967295] 00000400:00000200:1.0:1775776682.303070:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776682.303076:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776682.303080:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776682.303084:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776682.303087:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776682.303090:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775776682.303094:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776682.303097:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775776682.303107:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776682.303109:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776682.303112:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776682.303116:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776682.303119:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776682.303121:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776682.303123:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776682.303126:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449500 x1862022710432768/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776707 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.303162:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776682.303166:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449500 x1862022710432768/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776707 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.303173:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776682.303174:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776682.303178:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776682.303182:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776682.303185:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776682.303187:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308237327 00000100:00000001:1.0:1775776682.303190:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308241161 00000100:00000001:1.0:1775776682.303192:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308237327 00000100:00000001:1.0:1775776682.303195:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308241161 00000100:00000001:1.0:1775776682.303197:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776682.303199:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776682.303201:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776682.303204:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776682/real 1775776682] req@ffff8f6233449500 x1862022710432768/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776682.303213:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776682.303214:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776682.303216:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776682.303218:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776682.303220:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776682.303222:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776682.303225:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449500 x1862022710432768/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776682.303233:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449500 x1862022710432768/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.303239:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776682.303240:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776682.303242:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776682.303243:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776682.303247:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449500 x1862022710432768/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.303265:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776682.303269:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449500 x1862022710432768/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.303276:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776682.303279:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776682.303280:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776682.303282:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776682.303283:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776682.303289:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776682.303291:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776682.303295:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776682.303297:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776682.303301:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449500 x1862022710432768/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776682.303309:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710432768:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776682.303312:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776682.303315:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344ad80 x1862022710432896/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.303322:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776682.303323:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776682.303325:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776682.303327:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776682.303328:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776682.303331:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710432896:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776682.303334:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776682.303336:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776682.303338:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776682.303340:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776682.303343:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776682.303345:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776682.303348:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776682.303350:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776682.303353:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776682.303356:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710432896, portal 10 00000100:00000001:1.0:1775776682.303358:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776682.303359:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662546816 : -123823047004800 : ffff8f623344ad80) 00000100:00000040:1.0:1775776682.303364:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344ad80 x1862022710432896/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776707 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.303370:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776682.303372:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776682.303375:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776682.303377:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710432896, offset 0 00000400:00000200:1.0:1775776682.303381:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776682.303384:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776682.303388:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776682.303391:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776682.303393:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776682.303395:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776682.303397:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776682.303400:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453934:453934:256:4294967295] 192.168.201.115@tcp LPNI seq info [453859:453859:8:4294967295] 00000400:00000200:1.0:1775776682.303404:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776682.303410:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776682.303414:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776682.303417:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776682.303421:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776682.303423:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1775776682.303427:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776682.303429:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1775776682.303433:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776682.303435:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776682.303437:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776682.303440:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776682.303443:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776682.303445:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776682.303447:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776682.303450:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344ad80 x1862022710432896/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776707 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.303458:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776682.303460:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344ad80 x1862022710432896/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776707 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.303466:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776682.303468:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776682.303471:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776682.303473:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776682.303477:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776682.303478:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308237327 00000100:00000001:1.0:1775776682.303481:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308241161 00000100:00000001:1.0:1775776682.303482:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308237327 00000100:00000001:1.0:1775776682.303484:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308241161 00000100:00000001:1.0:1775776682.303486:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776682.303488:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776682.303490:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776682.303493:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776682/real 1775776682] req@ffff8f623344ad80 x1862022710432896/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776682.303501:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776682.303502:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776682.303504:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776682.303506:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776682.303507:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776682.303510:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776682.303512:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344ad80 x1862022710432896/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776682.303519:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344ad80 x1862022710432896/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.303525:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776682.303526:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776682.303528:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776682.303529:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776682.303533:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344ad80 x1862022710432896/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.303538:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776682.303543:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344ad80 x1862022710432896/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.303550:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776682.303552:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776682.303554:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776682.303555:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776682.303557:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776682.303558:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776682.303561:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776682.303564:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776682.303566:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776682.303571:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344ad80 x1862022710432896/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776682.303578:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710432896:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776682.303582:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776682.303583:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776682.303586:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344aa00 x1862022710432640/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.303593:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776682.303595:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776682.303597:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4d800. 02000000:00000001:1.0:1775776682.303600:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776682.303602:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776682.303603:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776682.303605:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776682.303608:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4e800. 02000000:00000001:1.0:1775776682.303611:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776682.303613:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776682.303615:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344aa00. 00000100:00000001:1.0:1775776682.303619:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776682.303620:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776682.303622:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776682.303624:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449500 x1862022710432768/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.303630:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776682.303631:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776682.303633:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776682.303635:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776682.303636:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776682.303637:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776682.303639:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776682.303640:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4f400. 02000000:00000001:1.0:1775776682.303642:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776682.303643:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776682.303644:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449500. 00000100:00000001:1.0:1775776682.303646:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776682.303647:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776682.303649:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776682.303651:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344ad80 x1862022710432896/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776707 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776682.303657:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776682.303658:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776682.303660:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776682.303661:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776682.303662:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776682.303663:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776682.303665:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776682.303666:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4c000. 02000000:00000001:1.0:1775776682.303668:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776682.303669:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776682.303670:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344ad80. 00000100:00000001:1.0:1775776682.303673:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776682.303674:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776682.303677:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776682.303683:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776682.303685:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776682.303687:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776682.303689:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776682.427273:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776682.427278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776682.427284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776682.427285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776682.427289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776682.427290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776682.427357:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776682.427361:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776682.427364:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776683.323383:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776683.323389:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776683.323391:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776683.323406:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776683.323410:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776683.323413:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776683.323416:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776683.323418:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776683.323420:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776683.451288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776683.451294:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776683.451299:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776683.451300:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776683.451304:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776683.451305:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776683.451371:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776683.451374:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776683.451377:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776684.347259:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776684.347273:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776684.347276:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776684.347281:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776684.347285:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776684.347288:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776684.347290:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776684.347292:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776684.347295:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776684.475243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776684.475249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776684.475254:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776684.475255:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776684.475259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776684.475260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776684.475322:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776684.475326:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776684.475329:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776685.371271:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776685.371278:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776685.371280:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776685.371286:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776685.371289:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776685.371293:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776685.371296:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776685.371298:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776685.371301:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776685.499240:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776685.499246:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776685.499253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776685.499254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776685.499257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776685.499258:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776685.499316:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776685.499319:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776685.499321:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776686.203244:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308241161, at 4308241216 00000100:00000040:1.0:1775776686.203244:0:214731:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308241161, at 4308241216 00000100:00000010:0.0:1775776686.203259:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e21e0. 00000100:00000010:1.0:1775776686.203261:0:214731:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e29c0. 00000100:00000001:2.0:1775776686.395239:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776686.395246:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776686.395248:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776686.395253:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776686.395256:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776686.395260:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776686.395262:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776686.395264:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776686.395266:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776686.523263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776686.523269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776686.523275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776686.523276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776686.523280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776686.523281:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776686.523334:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776686.523337:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776686.523339:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775776687.419285:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776687.419294:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776687.419296:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1775776687.419298:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1775776687.419300:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776687.419303:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776687.419305:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:3.0:1775776687.419306:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776687.419311:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00080000:2.0:1775776687.419314:0:141703:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13570 00000100:00000001:3.0:1775776687.419315:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776687.419319:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776687.419321:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776687.419324:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1775776687.419327:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776687.419329:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775776687.419336:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776687.419340:0:141703:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776687.419344:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776687.419348:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776687.419350:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776687.419353:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776687.419356:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776687.419358:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776687.419361:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776687.419364:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776687.419366:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776687.419369:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776687.419372:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776687.419374:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776687.419375:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776687.419377:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776687.419380:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776687.419383:0:141703:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776687.419385:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776687.419388:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776687.419391:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776687.419394:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776687.419396:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776687.419398:0:141703:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776687.419406:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449180. 00000020:00000040:2.0:1775776687.419410:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776687.419414:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776687.419415:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776687.419418:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776687.419420:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776687.419423:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776687.419430:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4e000. 00000100:00000001:2.0:1775776687.419436:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776687.419444:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449180 x1862022710433024/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776687.419468:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776687.419472:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776687.419477:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001c8e10ef] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449180 x1862022710433024/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776687.419495:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776687.419498:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776687.419501:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776687.419503:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776687.419504:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776687.419507:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776687.419509:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13570 00000400:00000200:2.0:1775776687.419513:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776687.419516:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776687.419519:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776687.419522:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776687.419525:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776687.419528:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776687.419530:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776687.419532:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776687.419534:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776687.419535:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1775776687.419537:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:2.0:1775776687.419537:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000100:00000001:1.0:1775776687.419542:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1775776687.419542:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000001:1.0:1775776687.419546:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1775776687.419546:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776687.419551:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776687.419554:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776687.419555:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449180 x1862022710433024/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775776687.419556:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776687.419560:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776687.419564:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776687.419570:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776687.419574:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776687.419577:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776687.419578:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776687.419580:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776687.419581:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776687.419584:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776687.419585:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776687.419588:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:1.0:1775776687.419589:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:2.0:1775776687.419592:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1775776687.419593:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1775776687.419598:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776687.419601:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710433024:192.168.201.115@tcp:250:kworker.0 00000100:00000001:2.0:1775776687.419602:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776687.419608:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000010:2.0:1775776687.419608:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775776687.419612:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776687.419613:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776687.419617:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776687.419618:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776687.419621:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776687.419622:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1775776687.419625:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776687.419628:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:1.0:1775776687.419629:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:2.0:1775776687.419630:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776687.419634:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776687.419638:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f000. 00000400:00000200:1.0:1775776687.419642:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:2.0:1775776687.419643:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776687.419648:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00100000:2.0:1775776687.419651:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710433152/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1775776687.419656:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000001:2.0:1775776687.419663:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:1.0:1775776687.419665:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710433024, portal 25 00000100:00000001:2.0:1775776687.419666:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776687.419671:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000040:2.0:1775776687.419673:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710433152/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.419674:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662539648 : -123823047011968 : ffff8f6233449180) 00000100:00000040:1.0:1775776687.419683:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449180 x1862022710433024/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776712 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776687.419691:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776687.419696:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1775776687.419697:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776687.419703:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00080000:2.0:1775776687.419703:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1775776687.419707:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:1.0:1775776687.419708:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00080000:2.0:1775776687.419710:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1775776687.419712:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710433024, offset 0 00000100:00000001:2.0:1775776687.419714:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776687.419720:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13570 00000400:00000200:1.0:1775776687.419722:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:2.0:1775776687.419727:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776687.419733:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776687.419734:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776687.419738:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776687.419741:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776687.419743:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000001:2.0:1775776687.419744:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776687.419748:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776687.419749:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776687.419751:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776687.419751:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:2.0:1775776687.419752:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776687.419755:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000400:00000200:1.0:1775776687.419758:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:2.0:1775776687.419758:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776687.419762:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776687.419765:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000400:00000200:1.0:1775776687.419766:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000001:2.0:1775776687.419767:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776687.419771:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776687.419773:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000200:1.0:1775776687.419774:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000020:00000040:2.0:1775776687.419775:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00080000:2.0:1775776687.419781:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000400:00000200:1.0:1775776687.419782:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000100:00000001:2.0:1775776687.419787:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776687.419791:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453935:453935:256:4294967295] 192.168.201.115@tcp LPNI seq info [453860:453860:8:4294967295] 02000000:00000001:2.0:1775776687.419791:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776687.419795:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1775776687.419801:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 02000000:00000001:2.0:1775776687.419801:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776687.419805:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776687.419807:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776687.419809:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776687.419814:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776687.419818:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000010:2.0:1775776687.419820:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775776687.419824:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000800:00000200:1.0:1775776687.419825:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 02000000:00000001:2.0:1775776687.419829:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776687.419831:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1775776687.419833:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000100:00000001:2.0:1775776687.419834:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776687.419837:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776687.419839:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1775776687.419844:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 02000000:00000010:2.0:1775776687.419845:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4ec00. 00000800:00000200:1.0:1775776687.419847:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1775776687.419851:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1775776687.419856:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000100:00100000:2.0:1775776687.419861:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710433280/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1775776687.419861:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775776687.419872:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000100:00000001:2.0:1775776687.419874:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000010:1.0:1775776687.419875:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000100:00000001:2.0:1775776687.419879:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776687.419883:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710433280/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1775776687.419887:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1775776687.419894:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776687.419898:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000100:00000040:2.0:1775776687.419899:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776687.419904:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1775776687.419905:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000100:00000040:2.0:1775776687.419907:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:1.0:1775776687.419910:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000040:2.0:1775776687.419912:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1775776687.419917:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:2.0:1775776687.419918:0:141703:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13580) 00000100:00000200:1.0:1775776687.419923:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449180 x1862022710433024/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776712 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.419939:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776687.419942:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449180 x1862022710433024/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776712 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.419949:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776687.419951:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776687.419955:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776687.419959:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776687.419962:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308246278 00000100:00000010:1.0:1775776687.419966:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e29c0. 00000100:00000001:1.0:1775776687.419972:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776687.419987:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308246278 00000100:00000010:1.0:1775776687.419994:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e22a0. 00000100:00000001:1.0:1775776687.419998:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776687.420002:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776687.420005:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776687.420008:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776687.420016:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776687/real 1775776687] req@ffff8f6233449180 x1862022710433024/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776687.420026:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776687.420027:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776687.420029:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776687.420032:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776687.420034:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776687.420037:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776687.420040:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449180 x1862022710433024/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776687.420048:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449180 x1862022710433024/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.420054:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776687.420056:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776687.420058:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776687.420060:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776687.420063:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449180 x1862022710433024/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.420070:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776687.420074:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449180 x1862022710433024/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.420080:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776687.420082:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776687.420084:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776687.420085:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776687.420087:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776687.420089:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776687.420092:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776687.420095:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776687.420097:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776687.420101:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449180 x1862022710433024/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776687.420108:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710433024:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776687.420112:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776687.420114:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776687.420116:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449180 x1862022710433024/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.420122:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776687.420124:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776687.420126:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776687.420129:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776687.420130:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776687.420132:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776687.420158:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776687.420161:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4e000. 02000000:00000001:1.0:1775776687.420176:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776687.420178:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776687.420180:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449180. 00000100:00000001:1.0:1775776687.420183:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776687.420184:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776687.420186:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776687.420192:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776687.420193:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776687.420195:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776687.420196:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776687.420198:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776687.420201:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449c00 x1862022710433152/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.420208:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776687.420209:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776687.420211:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776687.420213:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776687.420214:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776687.420217:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710433152:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776687.420220:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776687.420222:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776687.420223:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776687.420225:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776687.420228:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776687.420229:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776687.420233:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776687.420235:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776687.420239:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776687.420242:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710433152, portal 10 00000100:00000001:1.0:1775776687.420243:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776687.420245:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662542336 : -123823047009280 : ffff8f6233449c00) 00000100:00000040:1.0:1775776687.420251:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449c00 x1862022710433152/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776712 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.420257:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776687.420260:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776687.420262:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776687.420264:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710433152, offset 0 00000400:00000200:1.0:1775776687.420267:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776687.420272:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776687.420275:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776687.420277:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776687.420280:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776687.420282:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776687.420285:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776687.420288:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453936:453936:256:4294967295] 192.168.201.115@tcp LPNI seq info [453861:453861:8:4294967295] 00000400:00000200:1.0:1775776687.420292:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776687.420298:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776687.420302:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776687.420305:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776687.420309:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776687.420311:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775776687.420315:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776687.420318:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775776687.420321:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776687.420323:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776687.420325:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776687.420329:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776687.420331:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776687.420333:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776687.420336:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776687.420338:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449c00 x1862022710433152/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776712 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.420346:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776687.420348:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449c00 x1862022710433152/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776712 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.420360:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776687.420361:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776687.420364:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776687.420367:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776687.420371:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776687.420373:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308242444 00000100:00000001:1.0:1775776687.420375:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308246278 00000100:00000001:1.0:1775776687.420378:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308242444 00000100:00000001:1.0:1775776687.420379:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308246278 00000100:00000001:1.0:1775776687.420381:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776687.420383:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776687.420384:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776687.420388:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776687/real 1775776687] req@ffff8f6233449c00 x1862022710433152/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776687.420395:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776687.420397:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776687.420398:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776687.420401:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776687.420402:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776687.420404:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776687.420407:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449c00 x1862022710433152/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776687.420414:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449c00 x1862022710433152/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.420420:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776687.420422:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776687.420423:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776687.420425:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776687.420428:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449c00 x1862022710433152/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.420434:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776687.420438:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449c00 x1862022710433152/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.420444:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776687.420447:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776687.420449:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776687.420450:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776687.420451:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776687.420456:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776687.420458:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776687.420462:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776687.420464:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776687.420467:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449c00 x1862022710433152/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776687.420475:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710433152:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776687.420479:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776687.420482:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233448700 x1862022710433280/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.420488:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776687.420489:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776687.420491:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776687.420493:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776687.420495:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776687.420498:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710433280:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776687.420501:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776687.420503:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776687.420505:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776687.420506:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776687.420509:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776687.420511:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776687.420514:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776687.420516:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776687.420519:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776687.420522:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710433280, portal 10 00000100:00000001:1.0:1775776687.420524:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776687.420525:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662536960 : -123823047014656 : ffff8f6233448700) 00000100:00000040:1.0:1775776687.420530:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233448700 x1862022710433280/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776712 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.420535:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776687.420538:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776687.420540:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776687.420542:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710433280, offset 0 00000400:00000200:1.0:1775776687.420546:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776687.420550:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776687.420553:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776687.420556:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776687.420558:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776687.420560:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776687.420562:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776687.420565:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453937:453937:256:4294967295] 192.168.201.115@tcp LPNI seq info [453862:453862:8:4294967295] 00000400:00000200:1.0:1775776687.420569:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776687.420575:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776687.420580:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776687.420583:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776687.420587:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776687.420589:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1775776687.420594:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776687.420596:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1775776687.420600:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776687.420602:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776687.420605:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776687.420609:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776687.420611:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776687.420613:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776687.420616:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776687.420619:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233448700 x1862022710433280/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776712 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.420627:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776687.420629:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448700 x1862022710433280/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776712 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.420635:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776687.420636:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776687.420639:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776687.420641:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776687.420644:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776687.420646:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308242444 00000100:00000001:1.0:1775776687.420649:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308246278 00000100:00000001:1.0:1775776687.420651:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308242444 00000100:00000001:1.0:1775776687.420653:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308246278 00000100:00000001:1.0:1775776687.420655:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776687.420657:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776687.420659:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776687.420662:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776687/real 1775776687] req@ffff8f6233448700 x1862022710433280/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776687.420669:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776687.420670:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776687.420671:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776687.420673:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776687.420675:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776687.420677:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776687.420679:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233448700 x1862022710433280/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776687.420686:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233448700 x1862022710433280/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.420691:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776687.420693:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776687.420694:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776687.420695:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776687.420699:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233448700 x1862022710433280/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.420705:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776687.420708:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233448700 x1862022710433280/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.420714:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776687.420716:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776687.420717:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776687.420718:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776687.420720:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776687.420722:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776687.420724:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776687.420727:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776687.420729:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776687.420732:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233448700 x1862022710433280/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776687.420739:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710433280:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776687.420742:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776687.420745:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776687.420747:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449c00 x1862022710433152/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.420754:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776687.420755:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776687.420757:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776687.420759:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776687.420761:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776687.420762:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776687.420764:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776687.420765:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4f000. 02000000:00000001:1.0:1775776687.420768:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776687.420769:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776687.420770:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449c00. 00000100:00000001:1.0:1775776687.420772:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776687.420774:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776687.420775:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776687.420778:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233448700 x1862022710433280/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776712 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776687.420784:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776687.420785:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776687.420786:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776687.420788:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776687.420789:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776687.420790:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776687.420792:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776687.420793:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4ec00. 02000000:00000001:1.0:1775776687.420795:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776687.420796:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776687.420797:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233448700. 00000100:00000001:1.0:1775776687.420799:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776687.420800:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776687.420801:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776687.420804:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776687.420805:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776687.420807:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776687.420809:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776687.547279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776687.547285:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776687.547287:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776687.547290:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776687.547295:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776687.547297:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776687.547299:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776687.547301:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776687.547304:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776687.547306:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776687.547309:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776687.547310:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776687.547369:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776687.547373:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776687.547376:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776688.443205:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776688.443210:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776688.443212:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776688.443216:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776688.443218:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776688.443221:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776688.443223:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776688.443224:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776688.443226:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776688.571271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776688.571278:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776688.571284:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776688.571285:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776688.571289:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776688.571290:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776688.571351:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776688.571355:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776688.571358:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1775776689.467251:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1775776689.467258:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1775776689.467261:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776689.467266:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:3.0:1775776689.467270:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1775776689.467273:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776689.467276:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1775776689.467278:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776689.467280:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776689.595234:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776689.595239:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776689.595243:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776689.595244:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776689.595246:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776689.595247:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776689.595315:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776689.595319:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776689.595322:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776690.491234:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776690.491239:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776690.491240:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776690.491245:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776690.491248:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776690.491251:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776690.491253:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776690.491255:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776690.491257:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776690.619299:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776690.619305:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776690.619311:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776690.619312:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776690.619316:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776690.619317:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776690.619385:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776690.619388:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776690.619392:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776691.323246:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308246278, at 4308246336 00000100:00000010:0.0:1775776691.323258:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e29c0. 00000100:00000040:1.0:1775776691.323345:0:214731:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308246278, at 4308246336 00000100:00000010:1.0:1775776691.323349:0:214731:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e22a0. 00000100:00000001:2.0:1775776691.515328:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776691.515335:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776691.515337:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776691.515343:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776691.515346:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776691.515350:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776691.515353:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776691.515356:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776691.515358:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776691.643226:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776691.643232:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776691.643237:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776691.643240:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776691.643243:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776691.643247:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776691.643249:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776691.643257:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776691.643259:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776691.643261:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776691.643268:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776691.643269:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776691.643270:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776691.643271:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776691.643272:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776691.643273:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776691.643274:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776691.643275:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776691.643339:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776691.643342:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776691.643346:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775776692.539700:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775776692.539703:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776692.539710:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1775776692.539710:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776692.539714:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776692.539714:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1775776692.539720:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776692.539724:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1775776692.539727:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776692.539728:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1775776692.539733:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1775776692.539734:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776692.539738:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776692.539741:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776692.539743:0:181953:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13575 00000400:00000200:3.0:1775776692.539754:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776692.539760:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776692.539763:0:181953:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776692.539766:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776692.539770:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776692.539773:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776692.539775:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776692.539778:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776692.539780:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:3.0:1775776692.539783:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776692.539786:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:3.0:1775776692.539788:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:3.0:1775776692.539790:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:3.0:1775776692.539801:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776692.539803:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776692.539805:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776692.539807:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776692.539809:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:3.0:1775776692.539813:0:181953:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776692.539816:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776692.539819:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1775776692.539822:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776692.539825:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776692.539827:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:3.0:1775776692.539829:0:181953:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776692.539839:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72a00. 00000020:00000040:3.0:1775776692.539842:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:3.0:1775776692.539846:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776692.539848:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776692.539851:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776692.539853:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776692.539858:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1775776692.539865:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc36c00. 00000100:00000001:3.0:1775776692.539872:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776692.539889:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72a00 x1862022710433408/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1775776692.539898:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776692.539903:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776692.539911:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f70c1e86] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72a00 x1862022710433408/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1775776692.539934:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776692.539937:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776692.539940:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1775776692.539941:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776692.539943:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776692.539945:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1775776692.539947:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13575 00000400:00000200:3.0:1775776692.539950:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:3.0:1775776692.539952:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776692.539953:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776692.539955:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776692.539957:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776692.539958:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776692.539959:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776692.539961:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776692.539962:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:3.0:1775776692.539962:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776692.539964:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:3.0:1775776692.539965:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:3.0:1775776692.539966:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:3.0:1775776692.539968:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776692.539970:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1775776692.539971:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776692.539973:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:3.0:1775776692.539975:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:3.0:1775776692.539978:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776692.539980:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776692.539981:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1775776692.539982:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776692.539985:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776692.539988:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1775776692.539989:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776692.539992:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776692.539995:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000040:1.0:1775776692.539996:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72a00 x1862022710433408/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775776692.540000:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1775776692.540007:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776692.540011:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776692.540012:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776692.540015:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1775776692.540017:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c72300. 02000000:00000001:1.0:1775776692.540020:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:3.0:1775776692.540020:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776692.540024:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776692.540025:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1775776692.540027:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776692.540030:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776692.540030:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1775776692.540032:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776692.540035:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776692.540037:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710433408:192.168.201.115@tcp:250:kworker.0 02000000:00000010:3.0:1775776692.540039:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc34000. 00000100:00000001:3.0:1775776692.540044:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776692.540045:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776692.540050:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:3.0:1775776692.540051:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c72300 x1862022710433536/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1775776692.540054:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776692.540059:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1775776692.540060:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1775776692.540063:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776692.540065:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005051aecd] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c72300 x1862022710433536/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1775776692.540067:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776692.540071:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776692.540087:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1775776692.540090:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775776692.540094:0:181953:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:1.0:1775776692.540095:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776692.540096:0:181953:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1775776692.540099:0:181953:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1775776692.540101:0:181953:0:(import.c:484:import_select_connection()) Process entered 00000400:00000010:1.0:1775776692.540102:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00080000:3.0:1775776692.540104:0:181953:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13575 00000400:00000200:3.0:1775776692.540110:0:181953:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776692.540112:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000400:00000200:3.0:1775776692.540113:0:181953:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:3.0:1775776692.540118:0:181953:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1775776692.540121:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000200:1.0:1775776692.540123:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710433408, portal 25 00000100:00000001:3.0:1775776692.540124:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1775776692.540128:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000040:3.0:1775776692.540129:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776692.540131:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849014784 : -123823860536832 : ffff8f6202c72a00) 00000100:00000001:3.0:1775776692.540133:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:3.0:1775776692.540163:0:181953:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1775776692.540169:0:181953:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000020:00000001:3.0:1775776692.540172:0:181953:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1775776692.540174:0:181953:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000100:00000040:1.0:1775776692.540177:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72a00 x1862022710433408/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776717 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1775776692.540178:0:181953:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:3.0:1775776692.540183:0:181953:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:3.0:1775776692.540192:0:181953:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776692.540194:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1775776692.540195:0:181953:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1775776692.540200:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000100:00000040:3.0:1775776692.540201:0:181953:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:3.0:1775776692.540204:0:181953:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000400:00000010:1.0:1775776692.540205:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000020:00000040:3.0:1775776692.540207:0:181953:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00000200:1.0:1775776692.540209:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710433408, offset 0 00000100:00080000:3.0:1775776692.540212:0:181953:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:3.0:1775776692.540216:0:181953:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1775776692.540218:0:181953:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1775776692.540221:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 02000000:00000001:3.0:1775776692.540223:0:181953:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1775776692.540226:0:181953:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776692.540228:0:181953:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:3.0:1775776692.540230:0:181953:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1775776692.540231:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000008:00000001:3.0:1775776692.540236:0:181953:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776692.540238:0:181953:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776692.540241:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000010:3.0:1775776692.540244:0:181953:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6202c71180. 00000400:00000200:1.0:1775776692.540248:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:3.0:1775776692.540248:0:181953:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 02000000:00000001:3.0:1775776692.540253:0:181953:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1775776692.540256:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 02000000:00000001:3.0:1775776692.540256:0:181953:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1775776692.540260:0:181953:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1775776692.540263:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 02000000:00000001:3.0:1775776692.540263:0:181953:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1775776692.540267:0:181953:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776692.540269:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 02000000:00000010:3.0:1775776692.540273:0:181953:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f610fc35c00. 00000400:00000200:1.0:1775776692.540277:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453938:453938:256:4294967295] 192.168.201.115@tcp LPNI seq info [453863:453863:8:4294967295] 00000100:00000001:3.0:1775776692.540278:0:181953:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1775776692.540287:0:181953:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6202c71180 x1862022710433664/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1775776692.540288:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:3.0:1775776692.540299:0:181953:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1775776692.540303:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000100:00000001:3.0:1775776692.540304:0:181953:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776692.540309:0:181953:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7f9e068] to pc [ptlrpcd_rcv+-1] req@ffff8f6202c71180 x1862022710433664/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1775776692.540316:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000100:00000001:3.0:1775776692.540318:0:181953:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1775776692.540321:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:1.0:1775776692.540324:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000100:00000040:3.0:1775776692.540324:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776692.540325:0:181953:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1775776692.540329:0:181953:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1775776692.540333:0:181953:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13585) 00000800:00000200:1.0:1775776692.540336:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776692.540341:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775776692.540347:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776692.540351:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775776692.540355:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776692.540358:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776692.540364:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776692.540369:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776692.540374:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776692.540376:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776692.540382:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776692.540387:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72a00 x1862022710433408/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776717 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540396:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776692.540400:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72a00 x1862022710433408/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776717 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540406:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776692.540408:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776692.540413:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776692.540417:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776692.540419:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308251399 00000100:00000010:1.0:1775776692.540424:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e22a0. 00000100:00000001:1.0:1775776692.540430:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776692.540436:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308251399 00000100:00000010:1.0:1775776692.540439:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2360. 00000100:00000001:1.0:1775776692.540442:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776692.540445:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776692.540448:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776692.540450:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776692.540454:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776692/real 1775776692] req@ffff8f6202c72a00 x1862022710433408/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776692.540464:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776692.540465:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776692.540467:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776692.540468:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776692.540470:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776692.540471:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776692.540474:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72a00 x1862022710433408/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776692.540479:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72a00 x1862022710433408/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540484:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776692.540484:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776692.540486:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776692.540488:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776692.540490:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72a00 x1862022710433408/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540495:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776692.540498:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72a00 x1862022710433408/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540502:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776692.540504:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776692.540504:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776692.540505:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776692.540506:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776692.540508:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776692.540509:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776692.540512:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776692.540513:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776692.540515:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72a00 x1862022710433408/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776692.540520:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710433408:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776692.540523:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776692.540524:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776692.540525:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72a00 x1862022710433408/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540530:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776692.540532:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776692.540534:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776692.540536:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776692.540537:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776692.540538:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776692.540539:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776692.540541:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc36c00. 02000000:00000001:1.0:1775776692.540543:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776692.540545:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776692.540546:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72a00. 00000100:00000001:1.0:1775776692.540548:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776692.540549:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776692.540550:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776692.540556:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776692.540557:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776692.540558:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776692.540559:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776692.540560:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776692.540562:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c72300 x1862022710433536/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540566:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776692.540567:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776692.540568:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776692.540570:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776692.540571:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776692.540572:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710433536:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776692.540574:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776692.540575:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776692.540576:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776692.540577:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776692.540579:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776692.540580:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776692.540583:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776692.540584:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776692.540586:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776692.540588:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710433536, portal 10 00000100:00000001:1.0:1775776692.540589:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776692.540590:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849012992 : -123823860538624 : ffff8f6202c72300) 00000100:00000040:1.0:1775776692.540594:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c72300 x1862022710433536/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776717 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540598:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776692.540600:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776692.540601:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776692.540603:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710433536, offset 0 00000400:00000200:1.0:1775776692.540605:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776692.540608:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776692.540610:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776692.540612:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776692.540614:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776692.540615:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776692.540617:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776692.540619:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453939:453939:256:4294967295] 192.168.201.115@tcp LPNI seq info [453864:453864:8:4294967295] 00000400:00000200:1.0:1775776692.540621:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776692.540625:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776692.540628:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776692.540629:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776692.540632:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776692.540634:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775776692.540637:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776692.540638:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775776692.540641:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776692.540642:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776692.540644:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776692.540646:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776692.540648:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776692.540649:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776692.540651:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776692.540653:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c72300 x1862022710433536/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776717 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540657:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776692.540659:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c72300 x1862022710433536/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776717 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540663:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776692.540664:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776692.540666:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776692.540676:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776692.540680:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776692.540683:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308247564 00000100:00000001:1.0:1775776692.540685:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308251399 00000100:00000001:1.0:1775776692.540687:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308247564 00000100:00000001:1.0:1775776692.540689:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308251399 00000100:00000001:1.0:1775776692.540691:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776692.540693:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776692.540694:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776692.540698:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776692/real 1775776692] req@ffff8f6202c72300 x1862022710433536/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776692.540705:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776692.540706:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776692.540708:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776692.540710:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776692.540712:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776692.540714:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776692.540717:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c72300 x1862022710433536/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776692.540725:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c72300 x1862022710433536/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540732:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776692.540733:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776692.540743:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776692.540744:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776692.540746:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c72300 x1862022710433536/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540751:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776692.540753:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c72300 x1862022710433536/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540758:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776692.540760:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776692.540761:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776692.540762:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776692.540763:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776692.540766:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776692.540767:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776692.540769:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776692.540770:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776692.540772:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c72300 x1862022710433536/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776692.540777:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c72300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710433536:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776692.540779:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776692.540781:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6202c71180 x1862022710433664/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540785:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776692.540785:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776692.540787:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776692.540788:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776692.540789:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776692.540791:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710433664:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776692.540793:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776692.540794:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776692.540795:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776692.540796:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776692.540798:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776692.540799:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776692.540801:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776692.540803:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776692.540805:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776692.540806:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710433664, portal 10 00000100:00000001:1.0:1775776692.540808:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776692.540808:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620249849008512 : -123823860543104 : ffff8f6202c71180) 00000100:00000040:1.0:1775776692.540812:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6202c71180 x1862022710433664/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776717 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540816:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776692.540817:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776692.540818:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776692.540820:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710433664, offset 0 00000400:00000200:1.0:1775776692.540822:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776692.540824:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776692.540826:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776692.540828:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776692.540829:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776692.540831:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776692.540832:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776692.540834:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453940:453940:256:4294967295] 192.168.201.115@tcp LPNI seq info [453865:453865:8:4294967295] 00000400:00000200:1.0:1775776692.540837:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776692.540840:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776692.540843:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776692.540845:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776692.540847:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776692.540849:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1775776692.540851:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776692.540853:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1775776692.540861:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776692.540863:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776692.540866:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776692.540869:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776692.540871:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776692.540873:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776692.540876:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776692.540879:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6202c71180 x1862022710433664/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776717 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540886:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776692.540889:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6202c71180 x1862022710433664/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776717 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540895:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776692.540896:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776692.540899:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776692.540902:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776692.540905:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776692.540907:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308247564 00000100:00000001:1.0:1775776692.540909:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308251399 00000100:00000001:1.0:1775776692.540911:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308247564 00000100:00000001:1.0:1775776692.540913:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308251399 00000100:00000001:1.0:1775776692.540915:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776692.540917:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776692.540918:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776692.540921:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776692/real 1775776692] req@ffff8f6202c71180 x1862022710433664/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776692.540926:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776692.540927:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776692.540928:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776692.540930:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776692.540931:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776692.540932:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776692.540934:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6202c71180 x1862022710433664/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776692.540939:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6202c71180 x1862022710433664/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540943:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776692.540944:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776692.540945:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776692.540946:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776692.540948:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6202c71180 x1862022710433664/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540951:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776692.540953:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6202c71180 x1862022710433664/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540957:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776692.540958:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776692.540959:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776692.540959:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776692.540960:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776692.540962:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776692.540963:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776692.540965:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776692.540967:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776692.540969:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6202c71180 x1862022710433664/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776692.540973:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6202c71180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710433664:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776692.540975:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776692.540976:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776692.540977:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c72300 x1862022710433536/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.540981:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776692.540982:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776692.540983:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776692.540985:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776692.540986:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776692.540987:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776692.540988:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776692.540989:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc34000. 02000000:00000001:1.0:1775776692.540990:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776692.540991:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776692.540992:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c72300. 00000100:00000001:1.0:1775776692.540993:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776692.540994:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776692.540995:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776692.540997:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6202c71180 x1862022710433664/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776717 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776692.541000:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776692.541000:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776692.541001:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776692.541002:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776692.541003:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776692.541004:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776692.541004:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776692.541005:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f610fc35c00. 02000000:00000001:1.0:1775776692.541006:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776692.541007:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776692.541008:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6202c71180. 00000100:00000001:1.0:1775776692.541009:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776692.541009:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776692.541011:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776692.541013:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776692.541014:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776692.541015:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776692.541016:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776692.667267:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776692.667271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776692.667275:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776692.667276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776692.667279:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776692.667280:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776692.667343:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776692.667347:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776692.667350:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776693.563464:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776693.563471:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776693.563472:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776693.563476:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776693.563479:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776693.563481:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776693.563483:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776693.563485:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776693.563487:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776693.691263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776693.691269:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776693.691274:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776693.691276:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776693.691280:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776693.691282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776693.691344:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776693.691348:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776693.691350:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776694.587255:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776694.587263:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776694.587265:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776694.587270:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776694.587274:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776694.587278:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776694.587280:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776694.587282:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776694.587285:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776694.715255:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776694.715261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776694.715265:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776694.715267:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776694.715270:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776694.715271:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776694.715340:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776694.715344:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776694.715347:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776695.611286:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776695.611293:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776695.611295:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776695.611300:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776695.611304:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776695.611307:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776695.611310:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776695.611313:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776695.611316:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776695.739276:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776695.739282:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776695.739288:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776695.739289:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776695.739292:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776695.739292:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776695.739351:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776695.739354:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776695.739358:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1775776696.443262:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308251399, at 4308251456 00000100:00000040:1.0:1775776696.443262:0:214731:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308251399, at 4308251456 00000100:00000010:1.0:1775776696.443281:0:214731:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2360. 00000100:00000010:0.0:1775776696.443283:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e22a0. 00000100:00000001:2.0:1775776696.635285:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776696.635295:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776696.635297:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776696.635305:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776696.635310:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776696.635315:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776696.635318:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776696.635321:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776696.635323:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776696.763899:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776696.763908:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776696.763915:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776696.763917:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776696.763920:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776696.763922:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776696.764887:0:203108:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776696.764892:0:203108:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776696.764896:0:203108:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776697.659367:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776697.659376:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776697.659378:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776697.659384:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776697.659388:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776697.659392:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776697.659394:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776697.659397:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776697.659399:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776697.664157:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 46014e8d-88fb-49a9-96c8-7e89726047ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776697.664165:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775776697.664167:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776697.664170:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776697.664174:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776697.664180:0:141703:0:(import.c:509:import_select_connection()) MGC192.168.201.115@tcp: connect to NID 192.168.201.115@tcp last attempt 13580 00000400:00000200:2.0:1775776697.664188:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776697.664194:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776697.664196:0:141703:0:(import.c:588:import_select_connection()) MGC192.168.201.115@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776697.664199:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776697.664201:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776697.664203:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776697.664204:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776697.664207:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776697.664209:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c6070d5 00000020:00000001:2.0:1775776697.664211:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776697.664213:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f62113a6000 refcount=3 00000020:00000001:2.0:1775776697.664214:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000020:00000001:2.0:1775776697.664216:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620250091446272 : -123823618105344 : ffff8f62113a6000) 00000100:00000040:2.0:1775776697.664218:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776697.664219:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776697.664220:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776697.664221:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776697.664222:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f62113a6000 : new refcount 2 00000100:00080000:2.0:1775776697.664224:0:141703:0:(import.c:623:import_select_connection()) MGC192.168.201.115@tcp: import ffff8f622c04a000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776697.664225:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776697.664227:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776697.664230:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776697.664231:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776697.664233:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775776697.664234:0:141703:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776697.664243:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f623344bb80. 00000020:00000040:2.0:1775776697.664245:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f622c04a000 refcount=4 obd=MGC192.168.201.115@tcp 02000000:00000001:2.0:1775776697.664248:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776697.664249:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776697.664251:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776697.664253:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776697.664257:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776697.664263:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4d400. 00000100:00000001:2.0:1775776697.664267:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776697.664274:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f623344bb80 x1862022710433792/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776697.664281:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776697.664284:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775776697.664287:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000961938ea] to pc [ptlrpcd_rcv+-1] req@ffff8f623344bb80 x1862022710433792/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776697.664312:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776697.664315:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776697.664318:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775776697.664319:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776697.664321:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775776697.664323:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776697.664325:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13580 00000400:00000200:2.0:1775776697.664329:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776697.664331:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776697.664334:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775776697.664336:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776697.664338:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776697.664340:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776697.664342:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776697.664345:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776697.664347:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607129 00000020:00000001:2.0:1775776697.664349:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776697.664351:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b0f000 refcount=5 00000020:00000001:2.0:1775776697.664353:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000020:00000001:2.0:1775776697.664356:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914667008 : -123823794884608 : ffff8f6206b0f000) 00000100:00000040:2.0:1775776697.664359:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776697.664361:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776697.664363:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776697.664365:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776697.664367:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b0f000 : new refcount 4 00000100:00080000:2.0:1775776697.664371:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f62106eb800: import ffff8f6207a8c000 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776697.664373:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776697.664375:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1775776697.664381:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1775776697.664381:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776697.664390:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1775776697.664394:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776697.664395:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1775776697.664400:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776697.664406:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000040:1.0:1775776697.664411:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f623344bb80 x1862022710433792/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1775776697.664412:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776697.664423:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776697.664435:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776697.664438:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776697.664439:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233448700. 00000020:00000040:2.0:1775776697.664447:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a8c000 refcount=24 obd=lustre-MDT0000-mdc-ffff8f62106eb800 02000000:00000001:1.0:1775776697.664448:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775776697.664452:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776697.664453:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776697.664454:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775776697.664455:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775776697.664456:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775776697.664462:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4c000. 02000000:00000001:1.0:1775776697.664464:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775776697.664465:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776697.664473:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776697.664478:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233448700 x1862022710433920/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775776697.664485:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f623344bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710433792:192.168.201.115@tcp:250:kworker.0 00000100:00000001:2.0:1775776697.664497:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1775776697.664504:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1775776697.664505:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776697.664514:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776697.664518:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776697.664522:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:2.0:1775776697.664525:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c3833f86] to pc [ptlrpcd_rcv+-1] req@ffff8f6233448700 x1862022710433920/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776697.664534:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775776697.664536:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775776697.664539:0:141703:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000010:1.0:1775776697.664540:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 00000100:00000001:2.0:1775776697.664541:0:141703:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775776697.664544:0:141703:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1775776697.664545:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776697.664548:0:141703:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775776697.664552:0:141703:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: connect to NID 192.168.201.115@tcp last attempt 13580 00000400:00000200:1.0:1775776697.664566:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776697.664566:0:141703:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:2.0:1775776697.664571:0:141703:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.115@tcp is discovered: rc = 1 00000100:00080000:2.0:1775776697.664573:0:141703:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: tried all connections, increasing latency to 20s 00000400:00000010:1.0:1775776697.664576:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000100:00000040:2.0:1775776697.664581:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776697.664584:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775776697.664585:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776697.664586:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000001:2.0:1775776697.664588:0:141703:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775776697.664588:0:141703:0:(genops.c:823:class_conn2export()) looking for export cookie 0xf74d1861c607130 00000400:00000010:1.0:1775776697.664590:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000020:00000001:2.0:1775776697.664590:0:141703:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775776697.664594:0:141703:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8f6206b08800 refcount=5 00000020:00000001:2.0:1775776697.664596:0:141703:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000020:00000001:2.0:1775776697.664598:0:141703:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446620249914640384 : -123823794911232 : ffff8f6206b08800) 00000100:00000040:2.0:1775776697.664600:0:141703:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff8f62108fd440 refcount 14 to 192.168.201.115@tcp 00000100:00000001:2.0:1775776697.664601:0:141703:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1775776697.664602:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710433792, portal 25 00000100:00000040:2.0:1775776697.664603:0:141703:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8f62108fd440 refcount 15 to 192.168.201.115@tcp 00000100:00000001:1.0:1775776697.664610:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775776697.664610:0:141703:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446620250080269376 : -123823629282240 : ffff8f62108fd440) 00000020:00000040:2.0:1775776697.664613:0:141703:0:(genops.c:898:class_export_put()) PUTting export ffff8f6206b08800 : new refcount 4 00000100:00000001:1.0:1775776697.664615:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662550400 : -123823047001216 : ffff8f623344bb80) 00000100:00080000:2.0:1775776697.664617:0:141703:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff8f62106eb800: import ffff8f6207a89800 using connection 192.168.201.115@tcp/192.168.201.115@tcp 00000100:00000001:2.0:1775776697.664626:0:141703:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775776697.664627:0:141703:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775776697.664630:0:141703:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775776697.664631:0:141703:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776697.664632:0:141703:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775776697.664633:0:141703:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775776697.664634:0:141703:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776697.664635:0:141703:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775776697.664638:0:141703:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f6233449c00. 00000020:00000040:2.0:1775776697.664640:0:141703:0:(genops.c:1229:class_import_get()) import ffff8f6207a89800 refcount=6 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000100:00000040:1.0:1775776697.664644:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f623344bb80 x1862022710433792/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776722 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1775776697.664645:0:141703:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775776697.664656:0:141703:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775776697.664662:0:141703:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1775776697.664664:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1775776697.664664:0:141703:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1775776697.664672:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 02000000:00000001:2.0:1775776697.664672:0:141703:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1775776697.664676:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776697.664680:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862022710433792, offset 0 02000000:00000010:2.0:1775776697.664681:0:141703:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8f6207b4f400. 00000100:00000001:2.0:1775776697.664687:0:141703:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776697.664692:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000100:00100000:2.0:1775776697.664699:0:141703:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f6233449c00 x1862022710434048/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775776697.664710:0:141703:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775776697.664713:0:141703:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776697.664718:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000040:2.0:1775776697.664718:0:141703:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000170188f1] to pc [ptlrpcd_rcv+-1] req@ffff8f6233449c00 x1862022710434048/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775776697.664731:0:141703:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776697.664739:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000100:00000040:2.0:1775776697.664742:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776697.664747:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1775776697.664750:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1775776697.664751:0:141703:0:(pinger.c:229:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1775776697.664756:0:141703:0:(pinger.c:247:ptlrpc_pinger_process_import()) 57128860-4d5b-468e-b584-6d26ef8fb952->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1775776697.664760:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000100:00000040:2.0:1775776697.664760:0:141703:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (13590) 00000400:00000200:1.0:1775776697.664771:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776697.664775:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776697.664781:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453941:453941:256:4294967295] 192.168.201.115@tcp LPNI seq info [453866:453866:8:4294967295] 00000400:00000200:1.0:1775776697.664788:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776697.664799:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776697.664806:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776697.664813:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776697.664821:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776697.664825:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775776697.664831:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776697.664835:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1775776697.664839:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776697.664842:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776697.664850:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776697.664855:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776697.664861:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776697.664864:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776697.664871:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776697.664875:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f623344bb80 x1862022710433792/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776722 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.664885:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776697.664888:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f623344bb80 x1862022710433792/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 0 dl 1775776722 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.664893:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776697.664894:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776697.664901:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776697.664904:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776697.664907:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 0 is 4308256523 00000100:00000010:1.0:1775776697.664911:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2360. 00000100:00000001:1.0:1775776697.664918:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1775776697.664927:0:2411:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS new deadline estimation for cpu 1 is 4308256523 00000100:00000010:1.0:1775776697.664928:0:2411:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f62334e2420. 00000100:00000001:1.0:1775776697.664931:0:2411:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.115@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1775776697.664933:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776697.664935:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776697.664937:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776697.664940:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776697/real 1775776697] req@ffff8f623344bb80 x1862022710433792/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776697.664947:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776697.664948:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776697.664957:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776697.664960:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776697.664962:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776697.664965:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776697.664968:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f623344bb80 x1862022710433792/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776697.664976:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f623344bb80 x1862022710433792/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.664983:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776697.664998:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776697.665000:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776697.665002:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776697.665007:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f623344bb80 x1862022710433792/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.665015:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776697.665019:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f623344bb80 x1862022710433792/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.665025:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776697.665027:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776697.665028:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776697.665029:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776697.665031:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776697.665032:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776697.665035:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f622c04a000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776697.665038:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776697.665039:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776697.665042:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f623344bb80 x1862022710433792/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776697.665050:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f623344bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:46014e8d-88fb-49a9-96c8-7e89726047ea:2411:1862022710433792:192.168.201.115@tcp:250:kworker.0 00000100:00000001:1.0:1775776697.665056:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776697.665058:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776697.665061:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f623344bb80 x1862022710433792/t0(0) o250->MGC192.168.201.115@tcp@192.168.201.115@tcp:26/25 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.665067:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776697.665070:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776697.665073:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776697.665076:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776697.665078:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776697.665080:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f622c04a000 refcount=3 obd=MGC192.168.201.115@tcp 00000020:00000001:1.0:1775776697.665082:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776697.665084:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4d400. 02000000:00000001:1.0:1775776697.665088:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776697.665090:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776697.665092:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f623344bb80. 00000100:00000001:1.0:1775776697.665096:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776697.665097:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776697.665100:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776697.665109:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776697.665111:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776697.665114:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776697.665116:0:2411:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1775776697.665117:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776697.665121:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233448700 x1862022710433920/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.665127:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776697.665129:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776697.665131:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776697.665132:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776697.665178:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776697.665181:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710433920:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776697.665197:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776697.665199:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776697.665201:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776697.665203:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776697.665206:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776697.665208:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776697.665211:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776697.665215:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776697.665219:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776697.665221:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710433920, portal 10 00000100:00000001:1.0:1775776697.665223:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776697.665224:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662536960 : -123823047014656 : ffff8f6233448700) 00000100:00000040:1.0:1775776697.665240:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233448700 x1862022710433920/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776722 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.665247:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776697.665249:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776697.665251:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776697.665253:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710433920, offset 0 00000400:00000200:1.0:1775776697.665256:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776697.665260:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776697.665264:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776697.665267:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776697.665270:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776697.665272:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776697.665274:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776697.665277:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453942:453942:256:4294967295] 192.168.201.115@tcp LPNI seq info [453867:453867:8:4294967295] 00000400:00000200:1.0:1775776697.665281:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776697.665287:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776697.665292:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776697.665295:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776697.665298:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776697.665299:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775776697.665302:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776697.665305:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1775776697.665308:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776697.665310:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776697.665313:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776697.665316:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776697.665320:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776697.665322:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776697.665324:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776697.665328:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233448700 x1862022710433920/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776722 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.665335:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776697.665338:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233448700 x1862022710433920/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776722 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.665344:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776697.665345:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776697.665348:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776697.665352:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: using a timeout of 10373 usecs (11 jiffies) 00000400:00000200:1.0:1775776697.665355:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776697.665358:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308252689 00000100:00000001:1.0:1775776697.665361:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308256523 00000100:00000001:1.0:1775776697.665363:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308252689 00000100:00000001:1.0:1775776697.665365:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308256523 00000100:00000001:1.0:1775776697.665367:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776697.665369:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776697.665370:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776697.665374:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776697/real 1775776697] req@ffff8f6233448700 x1862022710433920/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776697.665380:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776697.665382:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776697.665383:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776697.665385:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776697.665386:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776697.665388:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776697.665391:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233448700 x1862022710433920/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776697.665398:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233448700 x1862022710433920/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.665404:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776697.665405:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776697.665407:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776697.665408:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776697.665412:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233448700 x1862022710433920/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.665417:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776697.665420:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233448700 x1862022710433920/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.665426:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776697.665428:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776697.665430:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776697.665431:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776697.665438:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776697.665445:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776697.665447:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a8c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776697.665450:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776697.665452:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776697.665456:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233448700 x1862022710433920/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776697.665463:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233448700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710433920:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776697.665467:0:2411:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1775776697.665470:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8f6233449c00 x1862022710434048/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.665476:0:2411:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1775776697.665478:0:2411:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776697.665480:0:2411:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1775776697.665481:0:2411:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1775776697.665483:0:2411:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1775776697.665485:0:2411:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710434048:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776697.665488:0:2411:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1775776697.665490:0:2411:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1775776697.665492:0:2411:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1775776697.665493:0:2411:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1775776697.665497:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776697.665498:0:2411:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1775776697.665501:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000010:1.0:1775776697.665503:0:2411:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f622e1fec08. 00000400:00000010:1.0:1775776697.665506:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3d48. 00000100:00000200:1.0:1775776697.665510:0:2411:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862022710434048, portal 10 00000100:00000001:1.0:1775776697.665511:0:2411:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1775776697.665513:0:2411:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446620250662542336 : -123823047009280 : ffff8f6233449c00) 00000100:00000040:1.0:1775776697.665518:0:2411:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff8f6233449c00 x1862022710434048/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776722 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.665524:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1775776697.665526:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.115@tcp 00000400:00000010:1.0:1775776697.665528:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f62258f3bb0. 00000100:00000200:1.0:1775776697.665530:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862022710434048, offset 0 00000400:00000200:1.0:1775776697.665534:0:2411:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.115@tcp 00000400:00000200:1.0:1775776697.665538:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776697.665542:0:2411:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.115@tcp local destination 00000400:00000200:1.0:1775776697.665544:0:2411:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1775776697.665546:0:2411:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.15@tcp 00000400:00000200:1.0:1775776697.665549:0:2411:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.15@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1775776697.665551:0:2411:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.115@tcp 00000400:00000200:1.0:1775776697.665554:0:2411:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.15@tcp NI seq info: [453943:453943:256:4294967295] 192.168.201.115@tcp LPNI seq info [453868:453868:8:4294967295] 00000400:00000200:1.0:1775776697.665558:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000400:00000200:1.0:1775776697.665563:0:2411:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.15@tcp(192.168.201.15@tcp:) -> 192.168.201.115@tcp(192.168.201.115@tcp:192.168.201.115@tcp) : PUT try# 0 00000800:00000200:1.0:1775776697.665567:0:2411:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776697.665570:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f6101230800. 00000800:00000200:1.0:1775776697.665574:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.0:1775776697.665576:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1775776697.665581:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f6202d89480] -> 12345-192.168.201.115@tcp (2) 00000800:00000200:1.1:1775776697.665583:0:2411:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.115 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1775776697.665586:0:2411:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.115@tcp 00000800:00000010:1.0:1775776697.665588:0:2411:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f6101230800. 00000400:00000200:1.0:1775776697.665591:0:2411:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1775776697.665594:0:2411:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.15@tcp->192.168.201.115@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1775776697.665597:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3bb0 00000400:00000010:1.0:1775776697.665599:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3bb0. 00000100:00000001:1.0:1775776697.665602:0:2411:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1775776697.665605:0:2411:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8f6233449c00 x1862022710434048/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776722 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.665619:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776697.665621:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8f6233449c00 x1862022710434048/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 0 dl 1775776722 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.665628:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776697.665629:0:2411:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1775776697.665632:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1775776697.665635:0:2411:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: using a timeout of 10742 usecs (11 jiffies) 00000400:00000200:1.0:1775776697.665638:0:2411:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.115@tcp to cpt 0 00000100:00000001:1.0:1775776697.665640:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 0 is 4308252689 00000100:00000001:1.0:1775776697.665642:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 0, keep current deadline 4308256523 00000100:00000001:1.0:1775776697.665644:0:2411:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS new deadline estimation for cpu 1 is 4308252689 00000100:00000001:1.0:1775776697.665646:0:2411:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8f62106eb800: PM QoS request active for cpu 1, keep current deadline 4308256523 00000100:00000001:1.0:1775776697.665648:0:2411:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1775776697.665651:0:2411:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776697.665652:0:2411:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1775776697.665656:0:2411:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775776697/real 1775776697] req@ffff8f6233449c00 x1862022710434048/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1775776697.665664:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1775776697.665665:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1775776697.665667:0:2411:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f622e1fec08. 00000400:00000200:1.0:1775776697.665669:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f62258f3d48 00000400:00000010:1.0:1775776697.665671:0:2411:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff8f62258f3d48. 00000100:00000001:1.0:1775776697.665672:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1775776697.665674:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8f6233449c00 x1862022710434048/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1775776697.665680:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8f6233449c00 x1862022710434048/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.666431:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1775776697.666436:0:2411:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776697.666438:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776697.666440:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1775776697.666446:0:2411:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f6233449c00 x1862022710434048/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.666454:0:2411:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1775776697.666457:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8f6233449c00 x1862022710434048/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.666465:0:2411:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1775776697.666467:0:2411:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776697.666469:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1775776697.666470:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776697.666473:0:2411:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1775776697.666474:0:2411:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1775776697.666477:0:2411:0:(import.c:65:import_set_state_nolock()) ffff8f6207a89800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775776697.666480:0:2411:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.115@tcp failed (-110) 00000100:00000001:1.0:1775776697.666483:0:2411:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1775776697.666487:0:2411:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8f6233449c00 x1862022710434048/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775776697.666494:0:2411:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f6233449c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:57128860-4d5b-468e-b584-6d26ef8fb952:2411:1862022710434048:192.168.201.115@tcp:38:kworker.0 00000100:00000001:1.0:1775776697.666499:0:2411:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776697.666501:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776697.666504:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233448700 x1862022710433920/t0(0) o38->lustre-MDT0000-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.666510:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776697.666512:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776697.666514:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4e400. 02000000:00000001:1.0:1775776697.666518:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776697.666520:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776697.666522:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a8c000 refcount=23 obd=lustre-MDT0000-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776697.666524:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776697.666526:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4c000. 02000000:00000001:1.0:1775776697.666529:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776697.666530:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776697.666532:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233448700. 00000100:00000001:1.0:1775776697.666534:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776697.666536:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776697.666538:0:2411:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1775776697.666540:0:2411:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8f6233449c00 x1862022710434048/t0(0) o38->lustre-MDT0001-mdc-ffff8f62106eb800@192.168.201.115@tcp:12/10 lens 520/544 e 0 to 1 dl 1775776722 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1775776697.666547:0:2411:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1775776697.666548:0:2411:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1775776697.666550:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8f622bf4f000. 02000000:00000001:1.0:1775776697.666552:0:2411:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1775776697.666553:0:2411:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1775776697.666554:0:2411:0:(genops.c:1242:class_import_put()) import ffff8f6207a89800 refcount=5 obd=lustre-MDT0001-mdc-ffff8f62106eb800 00000020:00000001:1.0:1775776697.666556:0:2411:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1775776697.666557:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8f6207b4f400. 02000000:00000001:1.0:1775776697.666560:0:2411:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1775776697.666561:0:2411:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1775776697.666562:0:2411:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f6233449c00. 00000100:00000001:1.0:1775776697.666565:0:2411:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1775776697.666566:0:2411:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776697.666569:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1775776697.666573:0:2411:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1775776697.666575:0:2411:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1775776697.666577:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1775776697.666579:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776697.787231:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776697.787238:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776697.787240:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776697.787244:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776697.787249:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776697.787251:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776697.787253:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776697.787255:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776697.787259:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776697.787260:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776697.787263:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776697.787264:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776697.787325:0:207610:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776697.787329:0:207610:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776697.787332:0:207610:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776698.683243:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776698.683248:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776698.683250:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776698.683254:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776698.683257:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776698.683259:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776698.683261:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776698.683263:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776698.683265:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776698.811244:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776698.811249:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776698.811253:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776698.811254:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776698.811257:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776698.811259:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775776698.811326:0:203109:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775776698.811331:0:203109:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775776698.811334:0:203109:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776699.707262:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776699.707271:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776699.707272:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776699.707279:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776699.707282:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776699.707286:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776699.707289:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776699.707291:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776699.707293:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776699.835309:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776699.835315:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776699.835320:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776699.835321:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776699.835332:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776699.835334:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776699.835394:0:207609:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776699.835397:0:207609:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776699.835400:0:207609:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1775776700.731265:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776700.731271:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776700.731273:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776700.731278:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776700.731282:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776700.731286:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776700.731288:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776700.731291:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776700.731293:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776700.859256:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776700.859261:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776700.859266:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776700.859268:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776700.859271:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776700.859273:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1775776700.859395:0:207611:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1775776700.859399:0:207611:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1775776700.859402:0:207611:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1775776701.563276:0:214731:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396910 (cpu 0) has reached its deadline 4308256523, at 4308256576 00000100:00000010:1.0:1775776701.563293:0:214731:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2420. 00000100:00000040:0.0:1775776701.565996:0:214685:0:(connection.c:75:cpu_latency_work()) work item of ffff8f6203396800 (cpu 0) has reached its deadline 4308256523, at 4308256578 00000100:00000010:0.0:1775776701.566009:0:214685:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f62334e2360. 00000100:00000001:2.0:1775776701.755237:0:214221:0:(client.c:2543:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1775776701.755241:0:214221:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775776701.755243:0:214221:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775776701.755247:0:214221:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f6233db5780 going to sleep for 0 seconds 00000100:00000001:2.0:1775776701.755250:0:214221:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775776701.755253:0:214221:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776701.755255:0:214221:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775776701.755257:0:214221:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775776701.755258:0:214221:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776701.883407:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776701.883413:0:214560:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776701.883418:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776701.883421:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776701.883423:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776701.883426:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776701.883428:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776701.883430:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776701.883432:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776701.883433:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776701.883436:0:214560:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1775776701.883437:0:214560:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1775776701.883438:0:214560:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1775776701.883440:0:214560:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776701.883441:0:214560:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1775776701.883442:0:214560:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776701.883443:0:214560:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1775776701.883445:0:214560:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) Debug log: 139014 lines, 139014 kept, 0 dropped, 0 bad.